Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-43693: Un-revert commits 2c1e258 and b2bf2bc. (gh-26577) 

These were reverted in gh-26530 (commit 17c4edc) due to refleaks.

* 2c1e258 - Compute deref offsets in compiler (gh-25152)
* b2bf2bc - Add new internal code objects fields: co_fastlocalnames and co_fastlocalkinds. (gh-26388)

This change fixes the refleaks.

https://bugs.python.org/issue43693
This commit is contained in:
Eric Snow 2021-06-07 12:22:26 -06:00 committed by GitHub
parent 001eb520b5
commit 2ab27c4af4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 6044 additions and 5710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Doc/library/dis.rst
View File

@ -1058,17 +1058,24 @@ All of the following opcodes use their arguments.
.. opcode:: LOAD_CLOSURE (i)
Pushes a reference to the cell contained in slot *i* of the cell and free
variable storage. The name of the variable is ``co_cellvars[i]`` if *i* is
less than the length of *co_cellvars*. Otherwise it is ``co_freevars[i -
len(co_cellvars)]``.
Pushes a reference to the cell contained in slot ``i`` of the "fast locals"
storage. The name of the variable is ``co_fastlocalnames[i]``.
Note that ``LOAD_CLOSURE`` is effectively an alias for ``LOAD_FAST``.
It exists to keep bytecode a little more readable.
.. versionchanged:: 3.11
``i`` is no longer offset by the length of ``co_varnames``.
.. opcode:: LOAD_DEREF (i)
Loads the cell contained in slot *i* of the cell and free variable storage.
Loads the cell contained in slot ``i`` of the "fast locals" storage.
Pushes a reference to the object the cell contains on the stack.
.. versionchanged:: 3.11
``i`` is no longer offset by the length of ``co_varnames``.
.. opcode:: LOAD_CLASSDEREF (i)
@ -1078,20 +1085,29 @@ All of the following opcodes use their arguments.
.. versionadded:: 3.4
.. versionchanged:: 3.11
``i`` is no longer offset by the length of ``co_varnames``.
.. opcode:: STORE_DEREF (i)
Stores TOS into the cell contained in slot *i* of the cell and free variable
Stores TOS into the cell contained in slot ``i`` of the "fast locals"
storage.
.. versionchanged:: 3.11
``i`` is no longer offset by the length of ``co_varnames``.
.. opcode:: DELETE_DEREF (i)
Empties the cell contained in slot *i* of the cell and free variable storage.
Empties the cell contained in slot ``i`` of the "fast locals" storage.
Used by the :keyword:`del` statement.
.. versionadded:: 3.2
.. versionchanged:: 3.11
``i`` is no longer offset by the length of ``co_varnames``.
.. opcode:: RAISE_VARARGS (argc)

24
Include/cpython/code.h
View File

@ -3,6 +3,8 @@
#endif
typedef uint16_t _Py_CODEUNIT;
// Each oparg must fit in the second half of _Py_CODEUNIT, hence 8 bits.
#define _Py_MAX_OPARG 255
#ifdef WORDS_BIGENDIAN
# define _Py_OPCODE(word) ((word) >> 8)
@ -16,6 +18,11 @@ typedef uint16_t _Py_CODEUNIT;
typedef struct _PyOpcache _PyOpcache;
// These are duplicated from pycore_code.h.
typedef unsigned char _PyLocalsPlusKind;
typedef _PyLocalsPlusKind *_PyLocalsPlusKinds;
/* Bytecode object */
struct PyCodeObject {
PyObject_HEAD
@ -47,7 +54,9 @@ struct PyCodeObject {
// The hottest fields (in the eval loop) are grouped here at the top.
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
_Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening */
_Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
Unlike the other "hot" fields, this one is
actually derived from co_code. */
PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
int co_flags; /* CO_..., see below */
int co_warmup; /* Warmup counter for quickening */
@ -59,9 +68,8 @@ struct PyCodeObject {
int co_stacksize; /* #entries needed for evaluation stack */
int co_firstlineno; /* first source line number */
PyObject *co_code; /* instruction opcodes */
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_localsplusnames; /* tuple mapping offsets to names */
_PyLocalsPlusKinds co_localspluskinds; /* array mapping to local kinds */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_linetable; /* string (encoding addr<->lineno mapping) See
@ -70,11 +78,15 @@ struct PyCodeObject {
/* These fields are set with computed values on new code objects. */
int *co_cell2arg; /* Maps cell vars which are arguments. */
// These are redundant but offer some performance benefit.
// redundant values (derived from co_localsplusnames and co_localspluskinds)
int co_nlocalsplus; /* number of local + cell + free variables */
int co_nlocals; /* number of local variables */
int co_ncellvars; /* number of cell variables */
int co_nfreevars; /* number of free variables */
// lazily-computed values
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
/* The remaining fields are zeroed out on new code objects. */
@ -152,7 +164,7 @@ struct PyCodeObject {
PyAPI_DATA(PyTypeObject) PyCode_Type;
#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
#define PyCode_GetNumFree(op) ((op)->co_nfreevars)
/* Public interface */
PyAPI_FUNC(PyCodeObject *) PyCode_New(

63
Include/internal/pycore_code.h
View File

@ -150,6 +150,58 @@ int _Py_Quicken(PyCodeObject *code);
extern Py_ssize_t _Py_QuickenedCount;
/* "Locals plus" for a code object is the set of locals + cell vars +
* free vars. This relates to variable names as well as offsets into
* the "fast locals" storage array of execution frames. The compiler
* builds the list of names, their offsets, and the corresponding
* kind of local.
*
* Those kinds represent the source of the initial value and the
* variable's scope (as related to closures). A "local" is an
* argument or other variable defined in the current scope. A "free"
* variable is one that is defined in an outer scope and comes from
* the function's closure. A "cell" variable is a local that escapes
* into an inner function as part of a closure, and thus must be
* wrapped in a cell. Any "local" can also be a "cell", but the
* "free" kind is mutually exclusive with both.
*/
// We would use an enum if C let us specify the storage type.
typedef unsigned char _PyLocalsPlusKind;
/* Note that these all fit within _PyLocalsPlusKind, as do combinations. */
// Later, we will use the smaller numbers to differentiate the different
// kinds of locals (e.g. pos-only arg, varkwargs, local-only).
#define CO_FAST_LOCAL 0x20
#define CO_FAST_CELL 0x40
#define CO_FAST_FREE 0x80
typedef _PyLocalsPlusKind *_PyLocalsPlusKinds;
static inline int
_PyCode_InitLocalsPlusKinds(int num, _PyLocalsPlusKinds *pkinds)
{
if (num == 0) {
*pkinds = NULL;
return 0;
}
_PyLocalsPlusKinds kinds = PyMem_NEW(_PyLocalsPlusKind, num);
if (kinds == NULL) {
PyErr_NoMemory();
return -1;
}
*pkinds = kinds;
return 0;
}
static inline void
_PyCode_ClearLocalsPlusKinds(_PyLocalsPlusKinds kinds)
{
if (kinds != NULL) {
PyMem_Free(kinds);
}
}
struct _PyCodeConstructor {
/* metadata */
PyObject *filename;
@ -166,13 +218,13 @@ struct _PyCodeConstructor {
PyObject *names;
/* mapping frame offsets to information */
PyObject *varnames;
PyObject *cellvars;
PyObject *freevars;
PyObject *localsplusnames;
_PyLocalsPlusKinds localspluskinds;
/* args (within varnames) */
int argcount;
int posonlyargcount;
// XXX Replace argcount with posorkwargcount (argcount - posonlyargcount).
int kwonlyargcount;
/* needed to create the frame */
@ -199,6 +251,11 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
int _PyCode_InitOpcache(PyCodeObject *co);
/* Getters for internal PyCodeObject data. */
PyAPI_FUNC(PyObject *) _PyCode_GetVarnames(PyCodeObject *);
PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *);
PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *);
#ifdef __cplusplus
}

6
Lib/ctypes/test/test_values.py
View File

@ -80,9 +80,9 @@ class PythonValuesTestCase(unittest.TestCase):
continue
items.append((entry.name.decode("ascii"), entry.size))
expected = [("__hello__", 138),
("__phello__", -138),
("__phello__.spam", 138),
expected = [("__hello__", 128),
("__phello__", -128),
("__phello__.spam", 128),
]
self.assertEqual(items, expected, "PyImport_FrozenModules example "
"in Doc/library/ctypes.rst may be out of date")

53
Lib/dis.py
View File

@ -273,15 +273,15 @@ def get_instructions(x, *, first_line=None):
the disassembled code object.
"""
co = _get_code_object(x)
cell_names = co.co_cellvars + co.co_freevars
linestarts = dict(findlinestarts(co))
if first_line is not None:
line_offset = first_line - co.co_firstlineno
else:
line_offset = 0
return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
co.co_consts, cell_names, linestarts,
line_offset)
return _get_instructions_bytes(co.co_code,
co._varname_from_oparg,
co.co_names, co.co_consts,
linestarts, line_offset)
def _get_const_info(const_index, const_list):
"""Helper to get optional details about const references
@ -295,7 +295,7 @@ def _get_const_info(const_index, const_list):
argval = const_list[const_index]
return argval, repr(argval)
def _get_name_info(name_index, name_list):
def _get_name_info(name_index, get_name, **extrainfo):
"""Helper to get optional details about named references
Returns the dereferenced name as both value and repr if the name
@ -303,8 +303,8 @@ def _get_name_info(name_index, name_list):
Otherwise returns the name index and its repr().
"""
argval = name_index
if name_list is not None:
argval = name_list[name_index]
if get_name is not None:
argval = get_name(name_index, **extrainfo)
argrepr = argval
else:
argrepr = repr(argval)
@ -336,8 +336,10 @@ def parse_exception_table(code):
except StopIteration:
return entries
def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
cells=None, linestarts=None, line_offset=0, exception_entries=()):
def _get_instructions_bytes(code, varname_from_oparg=None,
names=None, constants=None,
linestarts=None, line_offset=0,
exception_entries=()):
"""Iterate over the instructions in a bytecode string.
Generates a sequence of Instruction namedtuples giving the details of each
@ -346,6 +348,7 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
arguments.
"""
get_name = None if names is None else names.__getitem__
labels = set(findlabels(code))
for start, end, target, _, _ in exception_entries:
for i in range(start, end):
@ -368,20 +371,18 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
if op in hasconst:
argval, argrepr = _get_const_info(arg, constants)
elif op in hasname:
argval, argrepr = _get_name_info(arg, names)
argval, argrepr = _get_name_info(arg, get_name)
elif op in hasjabs:
argval = arg*2
argrepr = "to " + repr(argval)
elif op in hasjrel:
argval = offset + 2 + arg*2
argrepr = "to " + repr(argval)
elif op in haslocal:
argval, argrepr = _get_name_info(arg, varnames)
elif op in haslocal or op in hasfree:
argval, argrepr = _get_name_info(arg, varname_from_oparg)
elif op in hascompare:
argval = cmp_op[arg]
argrepr = argval
elif op in hasfree:
argval, argrepr = _get_name_info(arg, cells)
elif op == FORMAT_VALUE:
argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3]
argval = (argval, bool(arg & 0x4))
@ -398,11 +399,11 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
def disassemble(co, lasti=-1, *, file=None):
"""Disassemble a code object."""
cell_names = co.co_cellvars + co.co_freevars
linestarts = dict(findlinestarts(co))
exception_entries = parse_exception_table(co)
_disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
co.co_consts, cell_names, linestarts, file=file,
_disassemble_bytes(co.co_code, lasti,
co._varname_from_oparg,
co.co_names, co.co_consts, linestarts, file=file,
exception_entries=exception_entries)
def _disassemble_recursive(co, *, file=None, depth=None):
@ -416,8 +417,8 @@ def _disassemble_recursive(co, *, file=None, depth=None):
print("Disassembly of %r:" % (x,), file=file)
_disassemble_recursive(x, file=file, depth=depth)
def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
constants=None, cells=None, linestarts=None,
def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None,
names=None, constants=None, linestarts=None,
*, file=None, line_offset=0, exception_entries=()):
# Omit the line number column entirely if we have no line number info
show_lineno = bool(linestarts)
@ -434,8 +435,8 @@ def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
offset_width = len(str(maxoffset))
else:
offset_width = 4
for instr in _get_instructions_bytes(code, varnames, names,
constants, cells, linestarts,
for instr in _get_instructions_bytes(code, varname_from_oparg, names,
constants, linestarts,
line_offset=line_offset, exception_entries=exception_entries):
new_source_line = (show_lineno and
instr.starts_line is not None and
@ -517,7 +518,6 @@ class Bytecode:
else:
self.first_line = first_line
self._line_offset = first_line - co.co_firstlineno
self._cell_names = co.co_cellvars + co.co_freevars
self._linestarts = dict(findlinestarts(co))
self._original_object = x
self.current_offset = current_offset
@ -525,8 +525,9 @@ class Bytecode:
def __iter__(self):
co = self.codeobj
return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
co.co_consts, self._cell_names,
return _get_instructions_bytes(co.co_code,
co._varname_from_oparg,
co.co_names, co.co_consts,
self._linestarts,
line_offset=self._line_offset,
exception_entries=self.exception_entries)
@ -554,9 +555,9 @@ class Bytecode:
else:
offset = -1
with io.StringIO() as output:
_disassemble_bytes(co.co_code, varnames=co.co_varnames,
_disassemble_bytes(co.co_code,
varname_from_oparg=co._varname_from_oparg,
names=co.co_names, constants=co.co_consts,
cells=self._cell_names,
linestarts=self._linestarts,
line_offset=self._line_offset,
file=output,

4
Lib/importlib/_bootstrap_external.py
View File

@ -355,6 +355,8 @@ _code_type = type(_write_atomic.__code__)
# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling)
# Python 3.11a1 3451 (Add CALL_METHOD_KW)
# Python 3.11a1 3452 (drop nlocals from marshaled code objects)
# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
@ -364,7 +366,7 @@ _code_type = type(_write_atomic.__code__)
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
MAGIC_NUMBER = (3452).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3454).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'

40
Lib/test/test_dis.py
View File

@ -427,7 +427,7 @@ def _h(y):
return foo
dis_nested_0 = """\
%3d 0 LOAD_CLOSURE 0 (y)
%3d 0 LOAD_CLOSURE 2 (y)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo')
@ -444,12 +444,12 @@ dis_nested_0 = """\
dis_nested_1 = """%s
Disassembly of <code object foo at 0x..., file "%s", line %d>:
%3d 0 LOAD_CLOSURE 0 (x)
%3d 0 LOAD_CLOSURE 1 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>')
8 MAKE_FUNCTION 8 (closure)
10 LOAD_DEREF 1 (y)
10 LOAD_DEREF 2 (y)
12 GET_ITER
14 CALL_FUNCTION 1
16 RETURN_VALUE
@ -467,7 +467,7 @@ Disassembly of <code object <listcomp> at 0x..., file "%s", line %d>:
2 LOAD_FAST 0 (.0)
>> 4 FOR_ITER 6 (to 18)
6 STORE_FAST 1 (z)
8 LOAD_DEREF 0 (x)
8 LOAD_DEREF 2 (x)
10 LOAD_FAST 1 (z)
12 BINARY_ADD
14 LIST_APPEND 2
@ -962,16 +962,16 @@ expected_jumpy_line = 1
Instruction = dis.Instruction
expected_opinfo_outer = [
Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
@ -985,20 +985,20 @@ expected_opinfo_outer = [
expected_opinfo_f = [
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=5, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=6, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=6, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=34, starts_line=6, is_jump_target=False),
@ -1007,10 +1007,10 @@ expected_opinfo_f = [
expected_opinfo_inner = [
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='', offset=14, starts_line=None, is_jump_target=False),

4
Misc/NEWS.d/next/Core and Builtins/2021-04-02-15-02-16.bpo-43693.l3Ureu.rst Normal file
View File

@ -0,0 +1,4 @@
Compute cell offsets relative to locals in compiler. Allows the interpreter
to treats locals and cells a single array, which is slightly more efficient.
Also make the LOAD_CLOSURE opcode an alias for LOAD_FAST. Preserving
LOAD_CLOSURE helps keep bytecode a bit more readable.

3
Misc/NEWS.d/next/Core and Builtins/2021-05-21-20-53-49.bpo-43693.-NN3J_.rst Normal file
View File

@ -0,0 +1,3 @@
``PyCodeObject`` gained ``co_fastlocalnames`` and ``co_fastlocalkinds`` as
the the authoritative source of fast locals info. Marshaled code objects
have changed accordingly.

39
Objects/clinic/codeobject.c.h generated
View File

@ -373,4 +373,41 @@ skip_optional_kwonly:
exit:
return return_value;
}
/*[clinic end generated code: output=a272b22f63ea002e input=a9049054013a1b77]*/
PyDoc_STRVAR(code__varname_from_oparg__doc__,
"_varname_from_oparg($self, /, oparg)\n"
"--\n"
"\n"
"(internal-only) Return the local variable name for the given oparg.\n"
"\n"
"WARNING: this method is for internal use only and may change or go away.");
#define CODE__VARNAME_FROM_OPARG_METHODDEF \
{"_varname_from_oparg", (PyCFunction)(void(*)(void))code__varname_from_oparg, METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__},
static PyObject *
code__varname_from_oparg_impl(PyCodeObject *self, int oparg);
static PyObject *
code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
static const char * const _keywords[] = {"oparg", NULL};
static _PyArg_Parser _parser = {NULL, _keywords, "_varname_from_oparg", 0};
PyObject *argsbuf[1];
int oparg;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
if (!args) {
goto exit;
}
oparg = _PyLong_AsInt(args[0]);
if (oparg == -1 && PyErr_Occurred()) {
goto exit;
}
return_value = code__varname_from_oparg_impl(self, oparg);
exit:
return return_value;
}
/*[clinic end generated code: output=ba4c5487e0364ce8 input=a9049054013a1b77]*/

419
Objects/codeobject.c
View File

@ -154,6 +154,93 @@ validate_and_copy_tuple(PyObject *tup)
* _PyCode_New()
******************/
// This is also used in compile.c.
void
_Py_set_localsplus_info(int offset, PyObject *name, _PyLocalsPlusKind kind,
PyObject *names, _PyLocalsPlusKinds kinds)
{
Py_INCREF(name);
PyTuple_SET_ITEM(names, offset, name);
kinds[offset] = kind;
if (kind == CO_FAST_CELL) {
// Cells can overlap with args, so mark those cases.
int nlocalsplus = (int)PyTuple_GET_SIZE(names);
for (int i = 0; i < nlocalsplus; i++) {
_PyLocalsPlusKind kind = kinds[i];
if (kind && !(kind & CO_FAST_LOCAL)) {
// We've moved past the locals.
break;
}
PyObject *varname = PyTuple_GET_ITEM(names, i);
int cmp = PyUnicode_Compare(name, varname);
if (cmp == 0) {
kinds[i] |= CO_FAST_CELL;
break;
}
assert(cmp > 0 || !PyErr_Occurred());
}
}
}
static void
get_localsplus_counts(PyObject *names, _PyLocalsPlusKinds kinds,
int *pnlocals, int *pncellvars,
int *pnfreevars)
{
int nlocals = 0;
int ncellvars = 0;
int nfreevars = 0;
int nlocalsplus = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(names),
Py_ssize_t, int);
for (int i = 0; i < nlocalsplus; i++) {
if (kinds[i] & CO_FAST_LOCAL) {
nlocals += 1;
}
else if (kinds[i] & CO_FAST_CELL) {
ncellvars += 1;
}
else if (kinds[i] & CO_FAST_FREE) {
nfreevars += 1;
}
}
if (pnlocals != NULL) {
*pnlocals = nlocals;
}
if (pncellvars != NULL) {
*pncellvars = ncellvars;
}
if (pnfreevars != NULL) {
*pnfreevars = nfreevars;
}
}
static PyObject *
get_localsplus_names(PyCodeObject *co, _PyLocalsPlusKind kind, int num)
{
PyObject *names = PyTuple_New(num);
if (names == NULL) {
return NULL;
}
int index = 0;
for (int offset = 0; offset < co->co_nlocalsplus; offset++) {
if ((co->co_localspluskinds[offset] & kind) == 0) {
continue;
}
// For now there may be duplicates, which we ignore.
if (kind == CO_FAST_CELL && co->co_localspluskinds[offset] != kind) {
continue;
}
assert(index < num);
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, offset);
Py_INCREF(name);
PyTuple_SET_ITEM(names, index, name);
index += 1;
}
assert(index == num);
return names;
}
int
_PyCode_Validate(struct _PyCodeConstructor *con)
{
@ -164,9 +251,9 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
con->code == NULL || !PyBytes_Check(con->code) ||
con->consts == NULL || !PyTuple_Check(con->consts) ||
con->names == NULL || !PyTuple_Check(con->names) ||
con->varnames == NULL || !PyTuple_Check(con->varnames) ||
con->freevars == NULL || !PyTuple_Check(con->freevars) ||
con->cellvars == NULL || !PyTuple_Check(con->cellvars) ||
con->localsplusnames == NULL || !PyTuple_Check(con->localsplusnames) ||
(PyTuple_GET_SIZE(con->localsplusnames) && con->localspluskinds == NULL) ||
(!PyTuple_GET_SIZE(con->localsplusnames) && con->localspluskinds != NULL) ||
con->name == NULL || !PyUnicode_Check(con->name) ||
con->filename == NULL || !PyUnicode_Check(con->filename) ||
con->linetable == NULL || !PyBytes_Check(con->linetable) ||
@ -180,20 +267,30 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
a long running assumption in ceval.c and many parts of
the interpreter. */
if (PyBytes_GET_SIZE(con->code) > INT_MAX) {
PyErr_SetString(PyExc_OverflowError, "co_code larger than INT_MAX");
PyErr_SetString(PyExc_OverflowError,
"code: co_code larger than INT_MAX");
return -1;
}
if (PyBytes_GET_SIZE(con->code) % sizeof(_Py_CODEUNIT) != 0 ||
!_Py_IS_ALIGNED(PyBytes_AS_STRING(con->code), sizeof(_Py_CODEUNIT))
) {
PyErr_SetString(PyExc_ValueError, "code: co_code is malformed");
return -1;
}
/* Ensure that the co_varnames has enough names to cover the arg counts.
* Note that totalargs = nlocals - nplainlocals. We check nplainlocals
* here to avoid the possibility of overflow (however remote). */
int nplainlocals = (int)PyTuple_GET_SIZE(con->varnames) -
int nlocals;
get_localsplus_counts(con->localsplusnames, con->localspluskinds,
&nlocals, NULL, NULL);
int nplainlocals = nlocals -
con->argcount -
con->kwonlyargcount -
((con->flags & CO_VARARGS) != 0) -
((con->flags & CO_VARKEYWORDS) != 0);
if (nplainlocals < 0) {
PyErr_SetString(PyExc_ValueError, "code: varnames is too small");
PyErr_SetString(PyExc_ValueError, "code: co_varnames is too small");
return -1;
}
@ -203,6 +300,11 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
static void
init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
{
int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames);
int nlocals, ncellvars, nfreevars;
get_localsplus_counts(con->localsplusnames, con->localspluskinds,
&nlocals, &ncellvars, &nfreevars);
Py_INCREF(con->filename);
co->co_filename = con->filename;
Py_INCREF(con->name);
@ -221,12 +323,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
Py_INCREF(con->names);
co->co_names = con->names;
Py_INCREF(con->varnames);
co->co_varnames = con->varnames;
Py_INCREF(con->cellvars);
co->co_cellvars = con->cellvars;
Py_INCREF(con->freevars);
co->co_freevars = con->freevars;
Py_INCREF(con->localsplusnames);
co->co_localsplusnames = con->localsplusnames;
// We take ownership of the kinds array.
co->co_localspluskinds = con->localspluskinds;
co->co_argcount = con->argcount;
co->co_posonlyargcount = con->posonlyargcount;
@ -239,10 +339,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
/* derived values */
co->co_cell2arg = NULL; // This will be set soon.
co->co_nlocals = (int)PyTuple_GET_SIZE(con->varnames);
co->co_ncellvars = (int)PyTuple_GET_SIZE(con->cellvars);
co->co_nfreevars = (int)PyTuple_GET_SIZE(con->freevars);
co->co_nlocalsplus = co->co_nlocals + co->co_ncellvars + co->co_nfreevars;
co->co_nlocalsplus = nlocalsplus;
co->co_nlocals = nlocals;
co->co_ncellvars = ncellvars;
co->co_nfreevars = nfreevars;
co->co_varnames = NULL;
co->co_cellvars = NULL;
co->co_freevars = NULL;
/* not set */
co->co_weakreflist = NULL;
@ -274,23 +377,9 @@ _PyCode_New(struct _PyCodeConstructor *con)
if (intern_string_constants(con->consts, NULL) < 0) {
return NULL;
}
if (intern_strings(con->varnames) < 0) {
if (intern_strings(con->localsplusnames) < 0) {
return NULL;
}
if (intern_strings(con->freevars) < 0) {
return NULL;
}
if (intern_strings(con->cellvars) < 0) {
return NULL;
}
/* Check for any inner or outer closure references */
int ncellvars = (int)PyTuple_GET_SIZE(con->cellvars);
if (!ncellvars && !PyTuple_GET_SIZE(con->freevars)) {
con->flags |= CO_NOFREE;
} else {
con->flags &= ~CO_NOFREE;
}
PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type);
if (co == NULL) {
@ -299,18 +388,26 @@ _PyCode_New(struct _PyCodeConstructor *con)
}
init_code(co, con);
/* Check for any inner or outer closure references */
if (!co->co_ncellvars && !co->co_nfreevars) {
co->co_flags |= CO_NOFREE;
} else {
co->co_flags &= ~CO_NOFREE;
}
/* Create mapping between cells and arguments if needed. */
if (ncellvars) {
if (co->co_ncellvars) {
int totalargs = co->co_argcount +
co->co_kwonlyargcount +
((co->co_flags & CO_VARARGS) != 0) +
((co->co_flags & CO_VARKEYWORDS) != 0);
assert(totalargs <= co->co_nlocals);
/* Find cells which are also arguments. */
for (int i = 0; i < ncellvars; i++) {
PyObject *cellname = PyTuple_GET_ITEM(co->co_cellvars, i);
for (int i = 0; i < co->co_ncellvars; i++) {
PyObject *cellname = PyTuple_GET_ITEM(co->co_localsplusnames,
i + co->co_nlocals);
for (int j = 0; j < totalargs; j++) {
PyObject *argname = PyTuple_GET_ITEM(co->co_varnames, j);
PyObject *argname = PyTuple_GET_ITEM(co->co_localsplusnames, j);
int cmp = PyUnicode_Compare(cellname, argname);
if (cmp == -1 && PyErr_Occurred()) {
Py_DECREF(co);
@ -318,13 +415,13 @@ _PyCode_New(struct _PyCodeConstructor *con)
}
if (cmp == 0) {
if (co->co_cell2arg == NULL) {
co->co_cell2arg = PyMem_NEW(int, ncellvars);
co->co_cell2arg = PyMem_NEW(int, co->co_ncellvars);
if (co->co_cell2arg == NULL) {
Py_DECREF(co);
PyErr_NoMemory();
return NULL;
}
for (int k = 0; k < ncellvars; k++) {
for (int k = 0; k < co->co_ncellvars; k++) {
co->co_cell2arg[k] = CO_CELL_NOT_AN_ARG;
}
}
@ -352,6 +449,47 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
PyObject *filename, PyObject *name, int firstlineno,
PyObject *linetable, PyObject *exceptiontable)
{
PyCodeObject *co = NULL;
PyObject *localsplusnames = NULL;
_PyLocalsPlusKinds localspluskinds = NULL;
if (varnames == NULL || !PyTuple_Check(varnames) ||
cellvars == NULL || !PyTuple_Check(cellvars) ||
freevars == NULL || !PyTuple_Check(freevars)
) {
PyErr_BadInternalCall();
return NULL;
}
// Set the "fast locals plus" info.
int nvarnames = (int)PyTuple_GET_SIZE(varnames);
int ncellvars = (int)PyTuple_GET_SIZE(cellvars);
int nfreevars = (int)PyTuple_GET_SIZE(freevars);
int nlocalsplus = nvarnames + ncellvars + nfreevars;
localsplusnames = PyTuple_New(nlocalsplus);
if (localsplusnames == NULL) {
goto error;
}
if (_PyCode_InitLocalsPlusKinds(nlocalsplus, &localspluskinds) < 0) {
goto error;
}
int offset = 0;
for (int i = 0; i < nvarnames; i++, offset++) {
PyObject *name = PyTuple_GET_ITEM(varnames, i);
_Py_set_localsplus_info(offset, name, CO_FAST_LOCAL,
localsplusnames, localspluskinds);
}
for (int i = 0; i < ncellvars; i++, offset++) {
PyObject *name = PyTuple_GET_ITEM(cellvars, i);
_Py_set_localsplus_info(offset, name, CO_FAST_CELL,
localsplusnames, localspluskinds);
}
for (int i = 0; i < nfreevars; i++, offset++) {
PyObject *name = PyTuple_GET_ITEM(freevars, i);
_Py_set_localsplus_info(offset, name, CO_FAST_FREE,
localsplusnames, localspluskinds);
}
struct _PyCodeConstructor con = {
.filename = filename,
.name = name,
@ -364,9 +502,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
.consts = consts,
.names = names,
.varnames = varnames,
.cellvars = cellvars,
.freevars = freevars,
.localsplusnames = localsplusnames,
.localspluskinds = localspluskinds,
.argcount = argcount,
.posonlyargcount = posonlyargcount,
@ -376,18 +513,35 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
.exceptiontable = exceptiontable,
};
if (_PyCode_Validate(&con) < 0) {
return NULL;
goto error;
}
assert(PyBytes_GET_SIZE(code) % sizeof(_Py_CODEUNIT) == 0);
assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(code), sizeof(_Py_CODEUNIT)));
if (nlocals != PyTuple_GET_SIZE(varnames)) {
PyErr_SetString(PyExc_ValueError,
"code: co_nlocals != len(co_varnames)");
return NULL;
goto error;
}
return _PyCode_New(&con);
co = _PyCode_New(&con);
if (co == NULL) {
goto error;
}
localspluskinds = NULL; // This keeps it from getting freed below.
Py_INCREF(varnames);
co->co_varnames = varnames;
Py_INCREF(cellvars);
co->co_cellvars = cellvars;
Py_INCREF(freevars);
co->co_freevars = freevars;
error:
Py_XDECREF(localsplusnames);
_PyCode_ClearLocalsPlusKinds(localspluskinds);
return co;
}
PyCodeObject *
@ -438,9 +592,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
.linetable = emptystring,
.consts = nulltuple,
.names = nulltuple,
.varnames = nulltuple,
.cellvars = nulltuple,
.freevars = nulltuple,
.localsplusnames = nulltuple,
.exceptiontable = emptystring,
};
result = _PyCode_New(&con);
@ -883,6 +1035,56 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
}
/******************
* other PyCodeObject accessor functions
******************/
PyObject *
_PyCode_GetVarnames(PyCodeObject *co)
{
if (co->co_varnames == NULL) {
// PyCodeObject owns this reference.
co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL,
co->co_nlocals);
if (co->co_varnames == NULL) {
return NULL;
}
}
Py_INCREF(co->co_varnames);
return co->co_varnames;
}
PyObject *
_PyCode_GetCellvars(PyCodeObject *co)
{
if (co->co_cellvars == NULL) {
// PyCodeObject owns this reference.
co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL,
co->co_ncellvars);
if (co->co_cellvars == NULL) {
return NULL;
}
}
Py_INCREF(co->co_cellvars);
return co->co_cellvars;
}
PyObject *
_PyCode_GetFreevars(PyCodeObject *co)
{
if (co->co_freevars == NULL) {
// PyCodeObject owns this reference.
co->co_freevars = get_localsplus_names(co, CO_FAST_FREE,
co->co_nfreevars);
if (co->co_freevars == NULL) {
return NULL;
}
}
Py_INCREF(co->co_freevars);
return co->co_freevars;
}
/******************
* PyCode_Type
******************/
@ -1032,6 +1234,8 @@ code_dealloc(PyCodeObject *co)
Py_XDECREF(co->co_code);
Py_XDECREF(co->co_consts);
Py_XDECREF(co->co_names);
Py_XDECREF(co->co_localsplusnames);
_PyCode_ClearLocalsPlusKinds(co->co_localspluskinds);
Py_XDECREF(co->co_varnames);
Py_XDECREF(co->co_freevars);
Py_XDECREF(co->co_cellvars);
@ -1094,8 +1298,6 @@ code_richcompare(PyObject *self, PyObject *other, int op)
if (!eq) goto unequal;
eq = co->co_kwonlyargcount == cp->co_kwonlyargcount;
if (!eq) goto unequal;
eq = co->co_nlocals == cp->co_nlocals;
if (!eq) goto unequal;
eq = co->co_flags == cp->co_flags;
if (!eq) goto unequal;
eq = co->co_firstlineno == cp->co_firstlineno;
@ -1119,11 +1321,8 @@ code_richcompare(PyObject *self, PyObject *other, int op)
eq = PyObject_RichCompareBool(co->co_names, cp->co_names, Py_EQ);
if (eq <= 0) goto unequal;
eq = PyObject_RichCompareBool(co->co_varnames, cp->co_varnames, Py_EQ);
if (eq <= 0) goto unequal;
eq = PyObject_RichCompareBool(co->co_freevars, cp->co_freevars, Py_EQ);
if (eq <= 0) goto unequal;
eq = PyObject_RichCompareBool(co->co_cellvars, cp->co_cellvars, Py_EQ);
eq = PyObject_RichCompareBool(co->co_localsplusnames,
cp->co_localsplusnames, Py_EQ);
if (eq <= 0) goto unequal;
if (op == Py_EQ)
@ -1148,7 +1347,7 @@ code_richcompare(PyObject *self, PyObject *other, int op)
static Py_hash_t
code_hash(PyCodeObject *co)
{
Py_hash_t h, h0, h1, h2, h3, h4, h5, h6;
Py_hash_t h, h0, h1, h2, h3, h4;
h0 = PyObject_Hash(co->co_name);
if (h0 == -1) return -1;
h1 = PyObject_Hash(co->co_code);
@ -1157,15 +1356,11 @@ code_hash(PyCodeObject *co)
if (h2 == -1) return -1;
h3 = PyObject_Hash(co->co_names);
if (h3 == -1) return -1;
h4 = PyObject_Hash(co->co_varnames);
h4 = PyObject_Hash(co->co_localsplusnames);
if (h4 == -1) return -1;
h5 = PyObject_Hash(co->co_freevars);
if (h5 == -1) return -1;
h6 = PyObject_Hash(co->co_cellvars);
if (h6 == -1) return -1;
h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^
h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^
co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^
co->co_nlocals ^ co->co_flags;
co->co_flags;
if (h == -1) h = -2;
return h;
}
@ -1177,15 +1372,11 @@ static PyMemberDef code_memberlist[] = {
{"co_argcount", T_INT, OFF(co_argcount), READONLY},
{"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY},
{"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY},
{"co_nlocals", T_INT, OFF(co_nlocals), READONLY},
{"co_stacksize",T_INT, OFF(co_stacksize), READONLY},
{"co_flags", T_INT, OFF(co_flags), READONLY},
{"co_code", T_OBJECT, OFF(co_code), READONLY},
{"co_consts", T_OBJECT, OFF(co_consts), READONLY},
{"co_names", T_OBJECT, OFF(co_names), READONLY},
{"co_varnames", T_OBJECT, OFF(co_varnames), READONLY},
{"co_freevars", T_OBJECT, OFF(co_freevars), READONLY},
{"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY},
{"co_filename", T_OBJECT, OFF(co_filename), READONLY},
{"co_name", T_OBJECT, OFF(co_name), READONLY},
{"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
@ -1195,15 +1386,43 @@ static PyMemberDef code_memberlist[] = {
};
static PyObject *
code_getlnotab(PyCodeObject *code, void *closure)
{
return decode_linetable(code);
}
static PyObject *
code_getnlocals(PyCodeObject *code, void *closure)
{
return PyLong_FromLong(code->co_nlocals);
}
static PyObject *
code_getvarnames(PyCodeObject *code, void *closure)
{
return _PyCode_GetVarnames(code);
}
static PyObject *
code_getcellvars(PyCodeObject *code, void *closure)
{
return _PyCode_GetCellvars(code);
}
static PyObject *
code_getfreevars(PyCodeObject *code, void *closure)
{
return _PyCode_GetFreevars(code);
}
static PyGetSetDef code_getsetlist[] = {
{"co_lnotab", (getter)code_getlnotab, NULL, NULL},
// The following old names are kept for backward compatibility.
{"co_nlocals", (getter)code_getnlocals, NULL, NULL},
{"co_varnames", (getter)code_getvarnames, NULL, NULL},
{"co_cellvars", (getter)code_getcellvars, NULL, NULL},
{"co_freevars", (getter)code_getfreevars, NULL, NULL},
{0}
};
@ -1212,15 +1431,17 @@ static PyObject *
code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
{
Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co));
_PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
if (co->co_cell2arg != NULL && co->co_cellvars != NULL) {
res += co->co_ncellvars * sizeof(Py_ssize_t);
}
_PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
if (co_extra != NULL) {
res += sizeof(_PyCodeObjectExtra) +
(co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
}
if (co->co_cell2arg != NULL && co->co_cellvars != NULL) {
res += co->co_ncellvars * sizeof(Py_ssize_t);
}
if (co->co_opcache != NULL) {
assert(co->co_opcache_map != NULL);
// co_opcache_map
@ -1228,6 +1449,7 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
// co_opcache
res += co->co_opcache_size * sizeof(_PyOpcache);
}
return PyLong_FromSsize_t(res);
}
@ -1298,11 +1520,65 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
return NULL;
}
return (PyObject *)PyCode_NewWithPosOnlyArgs(
PyCodeObject *co = NULL;
PyObject *varnames = NULL;
PyObject *cellvars = NULL;
PyObject *freevars = NULL;
if (co_varnames == NULL) {
varnames = get_localsplus_names(self, CO_FAST_LOCAL, self->co_nlocals);
if (varnames == NULL) {
goto error;
}
co_varnames = varnames;
}
if (co_cellvars == NULL) {
cellvars = get_localsplus_names(self, CO_FAST_CELL, self->co_ncellvars);
if (cellvars == NULL) {
goto error;
}
co_cellvars = cellvars;
}
if (co_freevars == NULL) {
freevars = get_localsplus_names(self, CO_FAST_FREE, self->co_nfreevars);
if (freevars == NULL) {
goto error;
}
co_freevars = freevars;
}
co = PyCode_NewWithPosOnlyArgs(
co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals,
co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names,
co_varnames, co_freevars, co_cellvars, co_filename, co_name,
co_firstlineno, (PyObject*)co_linetable, (PyObject*)co_exceptiontable);
error:
Py_XDECREF(varnames);
Py_XDECREF(cellvars);
Py_XDECREF(freevars);
return (PyObject *)co;
}
/*[clinic input]
code._varname_from_oparg
oparg: int
(internal-only) Return the local variable name for the given oparg.
WARNING: this method is for internal use only and may change or go away.
[clinic start generated code]*/
static PyObject *
code__varname_from_oparg_impl(PyCodeObject *self, int oparg)
/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/
{
PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg);
if (name == NULL) {
return NULL;
}
Py_INCREF(name);
return name;
}
/* XXX code objects need to participate in GC? */
@ -1311,6 +1587,7 @@ static struct PyMethodDef code_methods[] = {
{"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS},
{"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS},
CODE_REPLACE_METHODDEF
CODE__VARNAME_FROM_OPARG_METHODDEF
{NULL, NULL} /* sentinel */
};

225
Objects/frameobject.c
View File

@ -4,6 +4,7 @@
#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_code.h" // CO_FAST_LOCAL, etc.
#include "frameobject.h" // PyFrameObject
#include "pycore_frame.h"
@ -659,9 +660,9 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
Py_VISIT(f->f_trace);
/* locals */
PyObject **fastlocals = f->f_localsptr;
for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
Py_VISIT(*fastlocals);
PyObject **localsplus = f->f_localsptr;
for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++localsplus) {
Py_VISIT(*localsplus);
}
/* stack */
@ -684,9 +685,9 @@ frame_tp_clear(PyFrameObject *f)
Py_CLEAR(f->f_trace);
/* locals */
PyObject **fastlocals = f->f_localsptr;
for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
Py_CLEAR(*fastlocals);
PyObject **localsplus = f->f_localsptr;
for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++localsplus) {
Py_CLEAR(*localsplus);
}
/* stack */
@ -917,112 +918,13 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
return f;
}
/* Convert between "fast" version of locals and dictionary version.
map and values are input arguments. map is a tuple of strings.
values is an array of PyObject*. At index i, map[i] is the name of
the variable with value values[i]. The function copies the first
nmap variable from map/values into dict. If values[i] is NULL,
the variable is deleted from dict.
If deref is true, then the values being copied are cell variables
and the value is extracted from the cell variable before being put
in dict.
*/
static int
map_to_dict(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values,
int deref)
{
Py_ssize_t j;
assert(PyTuple_Check(map));
assert(PyDict_Check(dict));
assert(PyTuple_Size(map) >= nmap);
for (j=0; j < nmap; j++) {
PyObject *key = PyTuple_GET_ITEM(map, j);
PyObject *value = values[j];
assert(PyUnicode_Check(key));
if (deref && value != NULL) {
assert(PyCell_Check(value));
value = PyCell_GET(value);
}
if (value == NULL) {
if (PyObject_DelItem(dict, key) != 0) {
if (PyErr_ExceptionMatches(PyExc_KeyError))
PyErr_Clear();
else
return -1;
}
}
else {
if (PyObject_SetItem(dict, key, value) != 0)
return -1;
}
}
return 0;
}
/* Copy values from the "locals" dict into the fast locals.
dict is an input argument containing string keys representing
variables names and arbitrary PyObject* as values.
map and values are input arguments. map is a tuple of strings.
values is an array of PyObject*. At index i, map[i] is the name of
the variable with value values[i]. The function copies the first
nmap variable from map/values into dict. If values[i] is NULL,
the variable is deleted from dict.
If deref is true, then the values being copied are cell variables
and the value is extracted from the cell variable before being put
in dict. If clear is true, then variables in map but not in dict
are set to NULL in map; if clear is false, variables missing in
dict are ignored.
Exceptions raised while modifying the dict are silently ignored,
because there is no good way to report them.
*/
static void
dict_to_map(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values,
int deref, int clear)
{
Py_ssize_t j;
assert(PyTuple_Check(map));
assert(PyDict_Check(dict));
assert(PyTuple_Size(map) >= nmap);
for (j=0; j < nmap; j++) {
PyObject *key = PyTuple_GET_ITEM(map, j);
PyObject *value = PyObject_GetItem(dict, key);
assert(PyUnicode_Check(key));
/* We only care about NULLs if clear is true. */
if (value == NULL) {
PyErr_Clear();
if (!clear)
continue;
}
if (deref) {
assert(PyCell_Check(values[j]));
if (PyCell_GET(values[j]) != value) {
if (PyCell_Set(values[j], value) < 0)
PyErr_Clear();
}
} else if (values[j] != value) {
Py_XINCREF(value);
Py_XSETREF(values[j], value);
}
Py_XDECREF(value);
}
}
int
PyFrame_FastToLocalsWithError(PyFrameObject *f)
{
/* Merge fast locals into f->f_locals */
PyObject *locals, *map;
PyObject *locals;
PyObject **fast;
PyCodeObject *co;
Py_ssize_t j;
if (f == NULL) {
PyErr_BadInternalCall();
@ -1035,25 +937,9 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
return -1;
}
co = f->f_code;
map = co->co_varnames;
if (!PyTuple_Check(map)) {
PyErr_Format(PyExc_SystemError,
"co_varnames must be a tuple, not %s",
Py_TYPE(map)->tp_name);
return -1;
}
fast = f->f_localsptr;
j = PyTuple_GET_SIZE(map);
if (j > co->co_nlocals)
j = co->co_nlocals;
if (co->co_nlocals) {
if (map_to_dict(map, j, locals, fast, 0) < 0)
return -1;
}
if (co->co_ncellvars || co->co_nfreevars) {
if (map_to_dict(co->co_cellvars, co->co_ncellvars,
locals, fast + co->co_nlocals, 1))
return -1;
for (int i = 0; i < co->co_nlocalsplus; i++) {
_PyLocalsPlusKind kind = co->co_localspluskinds[i];
/* If the namespace is unoptimized, then one of the
following cases applies:
@ -1063,12 +949,44 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
We don't want to accidentally copy free variables
into the locals dict used by the class.
*/
if (co->co_flags & CO_OPTIMIZED) {
if (map_to_dict(co->co_freevars, co->co_nfreevars, locals,
fast + co->co_nlocals + co->co_ncellvars, 1) < 0)
if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) {
continue;
}
/* Some args are also cells. For now each of those variables
has two indices in the fast array, with both marked as cells
but only one marked as an arg. That one is always set
to NULL in _PyEval_MakeFrameVector() and the other index
gets the cell holding the arg value. So we ignore the
former here and will later use the cell for the variable.
*/
if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
assert(fast[i] == NULL);
continue;
}
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
PyObject *value = fast[i];
if (kind & (CO_FAST_CELL | CO_FAST_FREE) && value != NULL) {
assert(PyCell_Check(value));
value = PyCell_GET(value);
}
if (value == NULL) {
if (PyObject_DelItem(locals, name) != 0) {
if (PyErr_ExceptionMatches(PyExc_KeyError)) {
PyErr_Clear();
}
else {
return -1;
}
}
}
else {
if (PyObject_SetItem(locals, name, value) != 0) {
return -1;
}
}
}
return 0;
}
@ -1088,36 +1006,51 @@ void
PyFrame_LocalsToFast(PyFrameObject *f, int clear)
{
/* Merge locals into fast locals */
PyObject *locals, *map;
PyObject *locals;
PyObject **fast;
PyObject *error_type, *error_value, *error_traceback;
PyCodeObject *co;
Py_ssize_t j;
if (f == NULL)
return;
locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
co = f->f_code;
map = co->co_varnames;
if (locals == NULL)
return;
if (!PyTuple_Check(map))
return;
PyErr_Fetch(&error_type, &error_value, &error_traceback);
fast = f->f_localsptr;
j = PyTuple_GET_SIZE(map);
if (j > co->co_nlocals)
j = co->co_nlocals;
if (co->co_nlocals)
dict_to_map(co->co_varnames, j, locals, fast, 0, clear);
if (co->co_ncellvars || co->co_nfreevars) {
dict_to_map(co->co_cellvars, co->co_ncellvars,
locals, fast + co->co_nlocals, 1, clear);
co = f->f_code;
PyErr_Fetch(&error_type, &error_value, &error_traceback);
for (int i = 0; i < co->co_nlocalsplus; i++) {
_PyLocalsPlusKind kind = co->co_localspluskinds[i];
/* Same test as in PyFrame_FastToLocals() above. */
if (co->co_flags & CO_OPTIMIZED) {
dict_to_map(co->co_freevars, co->co_nfreevars, locals,
fast + co->co_nlocals + co->co_ncellvars, 1,
clear);
if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) {
continue;
}
/* Same test as in PyFrame_FastToLocals() above. */
if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
continue;
}
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
PyObject *value = PyObject_GetItem(locals, name);
/* We only care about NULLs if clear is true. */
if (value == NULL) {
PyErr_Clear();
if (!clear) {
continue;
}
}
if (kind & (CO_FAST_CELL | CO_FAST_FREE)) {
assert(PyCell_Check(fast[i]));
if (PyCell_GET(fast[i]) != value) {
if (PyCell_Set(fast[i], value) < 0) {
PyErr_Clear();
}
}
} else if (fast[i] != value) {
Py_XINCREF(value);
Py_XSETREF(fast[i], value);
}
Py_XDECREF(value);
}
PyErr_Restore(error_type, error_value, error_traceback);
}

11
Objects/typeobject.c
View File

@ -2,6 +2,7 @@
#include "Python.h"
#include "pycore_call.h"
#include "pycore_code.h" // CO_FAST_FREE
#include "pycore_compile.h" // _Py_Mangle()
#include "pycore_initconfig.h"
#include "pycore_moduleobject.h" // _PyModule_GetDef()
@ -8894,13 +8895,15 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
return -1;
}
// Look for __class__ in the free vars.
PyTypeObject *type = NULL;
for (i = 0; i < co->co_nfreevars; i++) {
PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
i = co->co_nlocals + co->co_ncellvars;
for (; i < co->co_nlocalsplus; i++) {
assert(co->co_localspluskinds[i] & CO_FAST_FREE);
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
assert(PyUnicode_Check(name));
if (_PyUnicode_EqualToASCIIId(name, &PyId___class__)) {
Py_ssize_t index = co->co_nlocals + co->co_ncellvars + i;
PyObject *cell = f->f_localsptr[index];
PyObject *cell = f->f_localsptr[i];
if (cell == NULL || !PyCell_Check(cell)) {
PyErr_SetString(PyExc_RuntimeError,
"super(): bad __class__ cell");

11
Programs/test_frozenmain.h generated
View File

@ -19,10 +19,9 @@ unsigned char M_test_frozenmain[] = {
121,115,90,17,95,116,101,115,116,105,110,116,101,114,110,97,
108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114,
103,118,90,11,103,101,116,95,99,111,110,102,105,103,115,114,
2,0,0,0,218,3,107,101,121,169,0,114,8,0,0,0,
114,8,0,0,0,250,18,116,101,115,116,95,102,114,111,122,
101,110,109,97,105,110,46,112,121,218,8,60,109,111,100,117,
108,101,62,1,0,0,0,115,16,0,0,0,8,3,8,1,
8,2,12,1,12,1,8,1,26,7,4,249,243,0,0,0,
0,
2,0,0,0,218,3,107,101,121,169,0,250,18,116,101,115,
116,95,102,114,111,122,101,110,109,97,105,110,46,112,121,218,
8,60,109,111,100,117,108,101,62,1,0,0,0,115,16,0,
0,0,8,3,8,1,8,2,12,1,12,1,8,1,26,7,
4,249,243,0,0,0,0,
};

78
Python/ceval.c
View File

@ -1446,7 +1446,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
/* Local variable macros */
#define GETLOCAL(i) (fastlocals[i])
#define GETLOCAL(i) (localsplus[i])
/* The SETLOCAL() macro must not DECREF the local variable in-place and
then store the new value; it must copy the old value to a temporary
@ -1577,7 +1577,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
const _Py_CODEUNIT *next_instr;
int opcode; /* Current opcode */
int oparg; /* Current opcode argument, if any */
PyObject **fastlocals, **freevars, **specials;
PyObject **localsplus, **specials;
PyObject *retval = NULL; /* Return value */
_Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
PyCodeObject *co;
@ -1666,9 +1666,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
names = co->co_names;
consts = co->co_consts;
fastlocals = f->f_localsptr;
localsplus = f->f_localsptr;
first_instr = co->co_firstinstr;
freevars = f->f_localsptr + co->co_nlocals;
/*
f->f_lasti refers to the index of the last instruction,
unless it's -1 in which case next_instr should be first_instr.
@ -1836,12 +1835,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
DISPATCH();
}
/* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
case TARGET(LOAD_CLOSURE):
case TARGET(LOAD_FAST): {
PyObject *value = GETLOCAL(oparg);
if (value == NULL) {
format_exc_check_arg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(co->co_varnames, oparg));
PyTuple_GetItem(co->co_localsplusnames,
oparg));
goto error;
}
Py_INCREF(value);
@ -3069,13 +3071,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
format_exc_check_arg(
tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(co->co_varnames, oparg)
PyTuple_GetItem(co->co_localsplusnames, oparg)
);
goto error;
}
case TARGET(DELETE_DEREF): {
PyObject *cell = freevars[oparg];
PyObject *cell = GETLOCAL(oparg);
PyObject *oldobj = PyCell_GET(cell);
if (oldobj != NULL) {
PyCell_SET(cell, NULL);
@ -3086,21 +3088,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
goto error;
}
case TARGET(LOAD_CLOSURE): {
PyObject *cell = freevars[oparg];
Py_INCREF(cell);
PUSH(cell);
DISPATCH();
}
case TARGET(LOAD_CLASSDEREF): {
PyObject *name, *value, *locals = LOCALS();
Py_ssize_t idx;
assert(locals);
assert(oparg >= co->co_ncellvars);
idx = oparg - co->co_ncellvars;
assert(idx >= 0 && idx < co->co_nfreevars);
name = PyTuple_GET_ITEM(co->co_freevars, idx);
assert(oparg >= 0 && oparg < co->co_nlocalsplus);
name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
if (PyDict_CheckExact(locals)) {
value = PyDict_GetItemWithError(locals, name);
if (value != NULL) {
@ -3120,7 +3112,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
}
}
if (!value) {
PyObject *cell = freevars[oparg];
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
format_exc_unbound(tstate, co, oparg);
@ -3133,7 +3125,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
}
case TARGET(LOAD_DEREF): {
PyObject *cell = freevars[oparg];
PyObject *cell = GETLOCAL(oparg);
PyObject *value = PyCell_GET(cell);
if (value == NULL) {
format_exc_unbound(tstate, co, oparg);
@ -3146,7 +3138,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
case TARGET(STORE_DEREF): {
PyObject *v = POP();
PyObject *cell = freevars[oparg];
PyObject *cell = GETLOCAL(oparg);
PyObject *oldobj = PyCell_GET(cell);
PyCell_SET(cell, v);
Py_XDECREF(oldobj);
@ -4652,7 +4644,7 @@ format_missing(PyThreadState *tstate, const char *kind,
static void
missing_arguments(PyThreadState *tstate, PyCodeObject *co,
Py_ssize_t missing, Py_ssize_t defcount,
PyObject **fastlocals, PyObject *qualname)
PyObject **localsplus, PyObject *qualname)
{
Py_ssize_t i, j = 0;
Py_ssize_t start, end;
@ -4674,7 +4666,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co,
}
for (i = start; i < end; i++) {
if (GETLOCAL(i) == NULL) {
PyObject *raw = PyTuple_GET_ITEM(co->co_varnames, i);
PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i);
PyObject *name = PyObject_Repr(raw);
if (name == NULL) {
Py_DECREF(missing_names);
@ -4691,7 +4683,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co,
static void
too_many_positional(PyThreadState *tstate, PyCodeObject *co,
Py_ssize_t given, PyObject *defaults,
PyObject **fastlocals, PyObject *qualname)
PyObject **localsplus, PyObject *qualname)
{
int plural;
Py_ssize_t kwonly_given = 0;
@ -4755,7 +4747,7 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
PyObject* posonly_names = PyList_New(0);
for(int k=0; k < co->co_posonlyargcount; k++){
PyObject* posonly_name = PyTuple_GET_ITEM(co->co_varnames, k);
PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k);
for (int k2=0; k2<kwcount; k2++){
/* Compare the pointers first and fallback to PyObject_RichCompareBool*/
@ -4892,12 +4884,11 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i
static int
initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
PyObject **fastlocals, PyObject *const *args,
PyObject **localsplus, PyObject *const *args,
Py_ssize_t argcount, PyObject *kwnames)
{
PyCodeObject *co = (PyCodeObject*)con->fc_code;
const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
PyObject **freevars = fastlocals + co->co_nlocals;
/* Create a dictionary for keyword parameters (**kwags) */
PyObject *kwdict;
@ -4957,7 +4948,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
/* Speed hack: do raw pointer compares. As names are
normally interned this should almost always hit. */
co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item;
co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item;
for (j = co->co_posonlyargcount; j < total_args; j++) {
PyObject *varname = co_varnames[j];
if (varname == keyword) {
@ -5013,7 +5004,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
/* Check the number of positional arguments */
if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) {
too_many_positional(tstate, co, argcount, con->fc_defaults, fastlocals,
too_many_positional(tstate, co, argcount, con->fc_defaults, localsplus,
con->fc_qualname);
goto fail;
}
@ -5029,7 +5020,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
}
}
if (missing) {
missing_arguments(tstate, co, missing, defcount, fastlocals,
missing_arguments(tstate, co, missing, defcount, localsplus,
con->fc_qualname);
goto fail;
}
@ -5055,7 +5046,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
for (i = co->co_argcount; i < total_args; i++) {
if (GETLOCAL(i) != NULL)
continue;
PyObject *varname = PyTuple_GET_ITEM(co->co_varnames, i);
PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i);
if (con->fc_kwdefaults != NULL) {
PyObject *def = PyDict_GetItemWithError(con->fc_kwdefaults, varname);
if (def) {
@ -5070,12 +5061,13 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
missing++;
}
if (missing) {
missing_arguments(tstate, co, missing, -1, fastlocals,
missing_arguments(tstate, co, missing, -1, localsplus,
con->fc_qualname);
goto fail;
}
}
/* Allocate and initialize storage for cell vars, and copy free
vars into frame. */
for (i = 0; i < co->co_ncellvars; ++i) {
@ -5100,7 +5092,7 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
for (i = 0; i < co->co_nfreevars; ++i) {
PyObject *o = PyTuple_GET_ITEM(con->fc_closure, i);
Py_INCREF(o);
freevars[co->co_ncellvars + i] = o;
localsplus[co->co_nlocals + co->co_ncellvars + i] = o;
}
return 0;
@ -6433,15 +6425,11 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
/* Don't stomp existing exception */
if (_PyErr_Occurred(tstate))
return;
if (oparg < co->co_ncellvars) {
name = PyTuple_GET_ITEM(co->co_cellvars,
oparg);
format_exc_check_arg(tstate,
PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
name);
name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
if (oparg < co->co_ncellvars + co->co_nlocals) {
format_exc_check_arg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG, name);
} else {
name = PyTuple_GET_ITEM(co->co_freevars, oparg - co->co_ncellvars);
format_exc_check_arg(tstate, PyExc_NameError,
UNBOUNDFREE_ERROR_MSG, name);
}
@ -6483,16 +6471,14 @@ unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
switch (opcode) {
case STORE_FAST:
{
PyObject **fastlocals = f->f_localsptr;
PyObject **localsplus = f->f_localsptr;
if (GETLOCAL(oparg) == v)
SETLOCAL(oparg, NULL);
break;
}
case STORE_DEREF:
{
PyObject **freevars = (f->f_localsptr +
f->f_code->co_nlocals);
PyObject *c = freevars[oparg];
PyObject *c = f->f_localsptr[oparg];
if (PyCell_GET(c) == v) {
PyCell_SET(c, NULL);
Py_DECREF(v);

160
Python/compile.c
View File

@ -2047,16 +2047,16 @@ static int
compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags,
PyObject *qualname)
{
Py_ssize_t i, free = PyCode_GetNumFree(co);
if (qualname == NULL)
qualname = co->co_name;
if (free) {
for (i = 0; i < free; ++i) {
if (co->co_nfreevars) {
int i = co->co_nlocals + co->co_ncellvars;
for (; i < co->co_nlocalsplus; ++i) {
/* Bypass com_addop_varname because it will generate
LOAD_DEREF but LOAD_CLOSURE is needed.
*/
PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
/* Special case: If a class contains a method with a
free variable that has the same name as a method,
@ -2076,6 +2076,10 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags,
arg = compiler_lookup_arg(c->u->u_freevars, name);
}
if (arg == -1) {
PyObject *freevars = _PyCode_GetFreevars(co);
if (freevars == NULL) {
PyErr_Clear();
}
PyErr_Format(PyExc_SystemError,
"compiler_lookup_arg(name=%R) with reftype=%d failed in %S; "
"freevars of code %S: %R",
@ -2083,13 +2087,14 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags,
reftype,
c->u->u_name,
co->co_name,
co->co_freevars);
freevars);
Py_DECREF(freevars);
return 0;
}
ADDOP_I(c, LOAD_CLOSURE, arg);
}
flags |= 0x08;
ADDOP_I(c, BUILD_TUPLE, free);
ADDOP_I(c, BUILD_TUPLE, co->co_nfreevars);
}
ADDOP_LOAD_CONST(c, (PyObject*)co);
ADDOP_LOAD_CONST(c, qualname);
@ -7176,6 +7181,46 @@ merge_const_one(struct compiler *c, PyObject **obj)
return 1;
}
// This is in codeobject.c.
extern void _Py_set_localsplus_info(int, PyObject *, _PyLocalsPlusKind,
PyObject *, _PyLocalsPlusKinds);
static void
compute_localsplus_info(struct compiler *c,
PyObject *names, _PyLocalsPlusKinds kinds)
{
int nlocalsplus = (int)PyTuple_GET_SIZE(names);
PyObject *k, *v;
Py_ssize_t pos = 0;
while (PyDict_Next(c->u->u_varnames, &pos, &k, &v)) {
int offset = (int)PyLong_AS_LONG(v);
assert(offset >= 0);
assert(offset < nlocalsplus);
// For now we do not distinguish arg kinds.
_Py_set_localsplus_info(offset, k, CO_FAST_LOCAL, names, kinds);
}
int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames);
pos = 0;
while (PyDict_Next(c->u->u_cellvars, &pos, &k, &v)) {
int offset = (int)PyLong_AS_LONG(v);
assert(offset >= 0);
offset += nlocals;
assert(offset < nlocalsplus);
_Py_set_localsplus_info(offset, k, CO_FAST_CELL, names, kinds);
}
pos = 0;
while (PyDict_Next(c->u->u_freevars, &pos, &k, &v)) {
int offset = (int)PyLong_AS_LONG(v);
assert(offset >= 0);
offset += nlocals;
assert(offset < nlocalsplus);
_Py_set_localsplus_info(offset, k, CO_FAST_FREE, names, kinds);
}
}
static PyCodeObject *
makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
int maxdepth)
@ -7183,36 +7228,22 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
PyCodeObject *co = NULL;
PyObject *names = NULL;
PyObject *consts = NULL;
PyObject *varnames = NULL;
PyObject *localsplusnames = NULL;
_PyLocalsPlusKinds localspluskinds = NULL;
PyObject *name = NULL;
PyObject *freevars = NULL;
PyObject *cellvars = NULL;
int flags;
int posorkeywordargcount, posonlyargcount, kwonlyargcount;
names = dict_keys_inorder(c->u->u_names, 0);
varnames = dict_keys_inorder(c->u->u_varnames, 0);
if (!names || !varnames) {
if (!names) {
goto error;
}
cellvars = dict_keys_inorder(c->u->u_cellvars, 0);
if (!cellvars)
goto error;
freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_GET_SIZE(cellvars));
if (!freevars)
goto error;
if (!merge_const_one(c, &names) ||
!merge_const_one(c, &varnames) ||
!merge_const_one(c, &cellvars) ||
!merge_const_one(c, &freevars))
{
if (!merge_const_one(c, &names)) {
goto error;
}
flags = compute_code_flags(c);
if (flags < 0)
int flags = compute_code_flags(c);
if (flags < 0) {
goto error;
}
consts = PyList_AsTuple(constslist); /* PyCode_New requires a tuple */
if (consts == NULL) {
@ -7222,9 +7253,32 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
goto error;
}
posonlyargcount = Py_SAFE_DOWNCAST(c->u->u_posonlyargcount, Py_ssize_t, int);
posorkeywordargcount = Py_SAFE_DOWNCAST(c->u->u_argcount, Py_ssize_t, int);
kwonlyargcount = Py_SAFE_DOWNCAST(c->u->u_kwonlyargcount, Py_ssize_t, int);
assert(c->u->u_posonlyargcount < INT_MAX);
assert(c->u->u_argcount < INT_MAX);
assert(c->u->u_kwonlyargcount < INT_MAX);
int posonlyargcount = (int)c->u->u_posonlyargcount;
int posorkwargcount = (int)c->u->u_argcount;
assert(INT_MAX - posonlyargcount - posorkwargcount > 0);
int kwonlyargcount = (int)c->u->u_kwonlyargcount;
Py_ssize_t nlocals = PyDict_GET_SIZE(c->u->u_varnames);
Py_ssize_t ncellvars = PyDict_GET_SIZE(c->u->u_cellvars);
Py_ssize_t nfreevars = PyDict_GET_SIZE(c->u->u_freevars);
assert(nlocals < INT_MAX);
assert(ncellvars < INT_MAX);
assert(nfreevars < INT_MAX);
assert(INT_MAX - nlocals - ncellvars - nfreevars > 0);
int nlocalsplus = (int)nlocals + (int)ncellvars + (int)nfreevars;
localsplusnames = PyTuple_New(nlocalsplus);
if (localsplusnames == NULL) {
goto error;
}
if (_PyCode_InitLocalsPlusKinds(nlocalsplus, &localspluskinds) < 0) {
goto error;
}
compute_localsplus_info(c, localsplusnames, localspluskinds);
struct _PyCodeConstructor con = {
.filename = c->c_filename,
.name = c->u->u_name,
@ -7237,11 +7291,10 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
.consts = consts,
.names = names,
.varnames = varnames,
.cellvars = cellvars,
.freevars = freevars,
.localsplusnames = localsplusnames,
.localspluskinds = localspluskinds,
.argcount = posonlyargcount + posorkeywordargcount,
.argcount = posonlyargcount + posorkwargcount,
.posonlyargcount = posonlyargcount,
.kwonlyargcount = kwonlyargcount,
@ -7249,18 +7302,30 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
.exceptiontable = a->a_except_table,
};
if (_PyCode_Validate(&con) < 0) {
goto error;
}
if (!merge_const_one(c, &localsplusnames)) {
_PyCode_ClearLocalsPlusKinds(con.localspluskinds);
goto error;
}
con.localsplusnames = localsplusnames;
co = _PyCode_New(&con);
if (co == NULL) {
goto error;
}
localspluskinds = NULL; // This keeps it from getting freed below.
error:
Py_XDECREF(names);
Py_XDECREF(consts);
Py_XDECREF(varnames);
Py_XDECREF(localsplusnames);
_PyCode_ClearLocalsPlusKinds(localspluskinds);
Py_XDECREF(name);
Py_XDECREF(freevars);
Py_XDECREF(cellvars);
return co;
}
@ -7371,6 +7436,24 @@ guarantee_lineno_for_exits(struct assembler *a, int firstlineno) {
}
}
static void
offset_derefs(basicblock *entryblock, int nlocals)
{
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
for (int i = 0; i < b->b_iused; i++) {
struct instr *inst = &b->b_instr[i];
switch(inst->i_opcode) {
case LOAD_CLOSURE:
case LOAD_DEREF:
case STORE_DEREF:
case DELETE_DEREF:
case LOAD_CLASSDEREF:
inst->i_oparg += nlocals;
}
}
}
}
static PyCodeObject *
assemble(struct compiler *c, int addNone)
{
@ -7426,6 +7509,9 @@ assemble(struct compiler *c, int addNone)
a.a_entry = entryblock;
a.a_nblocks = nblocks;
assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX);
offset_derefs(entryblock, (int)PyDict_GET_SIZE(c->u->u_varnames));
consts = consts_dict_keys_inorder(c->u->u_consts);
if (consts == NULL) {
goto error;

7
Python/frozen_hello.h
View File

@ -5,8 +5,7 @@ const unsigned char _Py_M__hello[] = {
100,1,131,1,1,0,100,2,83,0,41,3,84,122,12,72,
101,108,108,111,32,119,111,114,108,100,33,78,41,2,90,11,
105,110,105,116,105,97,108,105,122,101,100,218,5,112,114,105,
110,116,169,0,114,1,0,0,0,114,1,0,0,0,122,14,
60,102,114,111,122,101,110,32,104,101,108,108,111,62,218,8,
60,109,111,100,117,108,101,62,1,0,0,0,115,4,0,0,
0,4,0,12,1,243,0,0,0,0,
110,116,169,0,122,14,60,102,114,111,122,101,110,32,104,101,
108,108,111,62,218,8,60,109,111,100,117,108,101,62,1,0,
0,0,115,4,0,0,0,4,0,12,1,243,0,0,0,0,
};

3232
Python/importlib.h generated
View File

File diff suppressed because it is too large Load Diff

5288
Python/importlib_external.h generated
View File

File diff suppressed because it is too large Load Diff

1985
Python/importlib_zipimport.h
View File

File diff suppressed because it is too large Load Diff

50
Python/marshal.c
View File

@ -518,9 +518,8 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
w_object(co->co_code, p);
w_object(co->co_consts, p);
w_object(co->co_names, p);
w_object(co->co_varnames, p);
w_object(co->co_freevars, p);
w_object(co->co_cellvars, p);
w_object(co->co_localsplusnames, p);
w_string(co->co_localspluskinds, co->co_nlocalsplus, p);
w_object(co->co_filename, p);
w_object(co->co_name, p);
w_long(co->co_firstlineno, p);
@ -1306,9 +1305,8 @@ r_object(RFILE *p)
PyObject *code = NULL;
PyObject *consts = NULL;
PyObject *names = NULL;
PyObject *varnames = NULL;
PyObject *freevars = NULL;
PyObject *cellvars = NULL;
PyObject *localsplusnames = NULL;
_PyLocalsPlusKinds localspluskinds = NULL;
PyObject *filename = NULL;
PyObject *name = NULL;
int firstlineno;
@ -1347,16 +1345,22 @@ r_object(RFILE *p)
names = r_object(p);
if (names == NULL)
goto code_error;
varnames = r_object(p);
if (varnames == NULL)
localsplusnames = r_object(p);
if (localsplusnames == NULL)
goto code_error;
Py_ssize_t nlocals = PyTuple_GET_SIZE(varnames);
freevars = r_object(p);
if (freevars == NULL)
goto code_error;
cellvars = r_object(p);
if (cellvars == NULL)
assert(PyTuple_GET_SIZE(localsplusnames) < INT_MAX);
int nlocalsplus = (int)PyTuple_GET_SIZE(localsplusnames);
if (nlocalsplus) {
if (_PyCode_InitLocalsPlusKinds(nlocalsplus,
&localspluskinds) < 0) {
goto code_error;
}
for (int i = 0; i < nlocalsplus; i++) {
localspluskinds[i] = r_byte(p);
}
}
filename = r_object(p);
if (filename == NULL)
goto code_error;
@ -1375,7 +1379,8 @@ r_object(RFILE *p)
if (PySys_Audit("code.__new__", "OOOiiiiii",
code, filename, name, argcount, posonlyargcount,
kwonlyargcount, nlocals, stacksize, flags) < 0) {
kwonlyargcount, nlocalsplus, stacksize,
flags) < 0) {
goto code_error;
}
@ -1391,9 +1396,8 @@ r_object(RFILE *p)
.consts = consts,
.names = names,
.varnames = varnames,
.cellvars = cellvars,
.freevars = freevars,
.localsplusnames = localsplusnames,
.localspluskinds = localspluskinds,
.argcount = argcount,
.posonlyargcount = posonlyargcount,
@ -1403,22 +1407,26 @@ r_object(RFILE *p)
.exceptiontable = exceptiontable,
};
if (_PyCode_Validate(&con) < 0) {
goto code_error;
}
v = (PyObject *)_PyCode_New(&con);
if (v == NULL) {
goto code_error;
}
localspluskinds = NULL; // This keeps it from getting freed below.
v = r_ref_insert(v, idx, flag, p);
code_error:
Py_XDECREF(code);
Py_XDECREF(consts);
Py_XDECREF(names);
Py_XDECREF(varnames);
Py_XDECREF(freevars);
Py_XDECREF(cellvars);
Py_XDECREF(localsplusnames);
_PyCode_ClearLocalsPlusKinds(localspluskinds);
Py_XDECREF(filename);
Py_XDECREF(name);
Py_XDECREF(linetable);

10
Python/suggestions.c
View File

@ -3,6 +3,7 @@
#include "pycore_frame.h"
#include "pycore_pyerrors.h"
#include "pycore_code.h" // _PyCode_GetVarnames()
#define MAX_CANDIDATE_ITEMS 750
#define MAX_STRING_SIZE 40
@ -210,8 +211,13 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc)
PyFrameObject *frame = traceback->tb_frame;
assert(frame != NULL);
PyCodeObject *code = PyFrame_GetCode(frame);
assert(code != NULL && code->co_varnames != NULL);
PyObject *dir = PySequence_List(code->co_varnames);
assert(code != NULL && code->co_localsplusnames != NULL);
PyObject *varnames = _PyCode_GetVarnames(code);
if (varnames == NULL) {
return NULL;
}
PyObject *dir = PySequence_List(varnames);
Py_DECREF(varnames);
Py_DECREF(code);
if (dir == NULL) {
return NULL;

8
Tools/gdb/libpython.py
View File

@ -871,7 +871,8 @@ class PyFrameObjectPtr(PyObjectPtr):
self.f_lineno = int_from_int(self.field('f_lineno'))
self.f_lasti = int_from_int(self.field('f_lasti'))
self.co_nlocals = int_from_int(self.co.field('co_nlocals'))
self.co_varnames = PyTupleObjectPtr.from_pyobject_ptr(self.co.field('co_varnames'))
pnames = self.co.field('co_localsplusnames')
self.co_localsplusnames = PyTupleObjectPtr.from_pyobject_ptr(pnames)
def iter_locals(self):
'''
@ -884,8 +885,9 @@ class PyFrameObjectPtr(PyObjectPtr):
f_localsplus = self.field('f_localsptr')
for i in safe_range(self.co_nlocals):
pyop_value = PyObjectPtr.from_pyobject_ptr(f_localsplus[i])
if not pyop_value.is_null():
pyop_name = PyObjectPtr.from_pyobject_ptr(self.co_varnames[i])
if pyop_value.is_null():
continue
pyop_name = PyObjectPtr.from_pyobject_ptr(self.co_localsplusnames[i])
yield (pyop_name, pyop_value)
def _f_globals(self):