Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-43693: Un-revert commit f3fa63e. (#26609) 

This was reverted in GH-26596 (commit 6d518bb) due to some bad memory accesses.

* Add the MAKE_CELL opcode. (gh-26396)

The memory accesses have been fixed.

https://bugs.python.org/issue43693
This commit is contained in:
Eric Snow 2021-06-08 16:01:34 -06:00 committed by GitHub
parent ab36b9f834
commit 3e1c7167d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 4470 additions and 4234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Doc/library/dis.rst
View File

@ -1056,6 +1056,14 @@ All of the following opcodes use their arguments.
Deletes local ``co_varnames[var_num]``.
.. opcode:: MAKE_CELL (i)
Creates a new cell in slot ``i``. If that slot is empty then
that value is stored into the new cell.
.. versionadded:: 3.11
.. opcode:: LOAD_CLOSURE (i)
Pushes a reference to the cell contained in slot ``i`` of the "fast locals"

11
Include/cpython/code.h
View File

@ -2,9 +2,16 @@
# error "this header file must not be included directly"
#endif
/* Each instruction in a code object is a fixed-width value,
* currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG
* opcode allows for larger values but the current limit is 3 uses
* of EXTENDED_ARG (see Python/wordcode_helpers.h), for a maximum
* 32-bit value. This aligns with the note in Python/compile.c
* (compiler_addop_i_line) indicating that the max oparg value is
* 2**32 - 1, rather than INT_MAX.
*/
typedef uint16_t _Py_CODEUNIT;
// Each oparg must fit in the second half of _Py_CODEUNIT, hence 8 bits.
#define _Py_MAX_OPARG 255
#ifdef WORDS_BIGENDIAN
# define _Py_OPCODE(word) ((word) >> 8)

2
Include/internal/pycore_frame.h
View File

@ -32,6 +32,8 @@ _PyFrame_GetBuiltins(PyFrameObject *f)
int _PyFrame_TakeLocals(PyFrameObject *f);
PyAPI_FUNC(int) _PyFrame_OpAlreadyRan(PyFrameObject *f, int opcode, int oparg);
#ifdef __cplusplus
}
#endif

9
Include/opcode.h generated
View File

@ -113,10 +113,11 @@ extern "C" {
#define CALL_FUNCTION 131
#define MAKE_FUNCTION 132
#define BUILD_SLICE 133
#define LOAD_CLOSURE 135
#define LOAD_DEREF 136
#define STORE_DEREF 137
#define DELETE_DEREF 138
#define MAKE_CELL 135
#define LOAD_CLOSURE 136
#define LOAD_DEREF 137
#define STORE_DEREF 138
#define DELETE_DEREF 139
#define CALL_FUNCTION_KW 141
#define CALL_FUNCTION_EX 142
#define EXTENDED_ARG 144

3
Lib/importlib/_bootstrap_external.py
View File

@ -357,6 +357,7 @@ _code_type = type(_write_atomic.__code__)
# Python 3.11a1 3452 (drop nlocals from marshaled code objects)
# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
# Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
@ -366,7 +367,7 @@ _code_type = type(_write_atomic.__code__)
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
MAGIC_NUMBER = (3454).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3455).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'

10
Lib/opcode.py
View File

@ -181,14 +181,16 @@ def_op('CALL_FUNCTION', 131) # #args
def_op('MAKE_FUNCTION', 132) # Flags
def_op('BUILD_SLICE', 133) # Number of items
def_op('LOAD_CLOSURE', 135)
def_op('MAKE_CELL', 135)
hasfree.append(135)
def_op('LOAD_DEREF', 136)
def_op('LOAD_CLOSURE', 136)
hasfree.append(136)
def_op('STORE_DEREF', 137)
def_op('LOAD_DEREF', 137)
hasfree.append(137)
def_op('DELETE_DEREF', 138)
def_op('STORE_DEREF', 138)
hasfree.append(138)
def_op('DELETE_DEREF', 139)
hasfree.append(139)
def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs
def_op('CALL_FUNCTION_EX', 142) # Flags

129
Lib/test/test_dis.py
View File

@ -427,15 +427,17 @@ def _h(y):
return foo
dis_nested_0 = """\
%3d 0 LOAD_CLOSURE 2 (y)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo')
8 MAKE_FUNCTION 8 (closure)
10 STORE_FAST 1 (foo)
0 MAKE_CELL 2 (y)
%3d 12 LOAD_FAST 1 (foo)
14 RETURN_VALUE
%3d 2 LOAD_CLOSURE 2 (y)
4 BUILD_TUPLE 1
6 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>)
8 LOAD_CONST 2 ('_h.<locals>.foo')
10 MAKE_FUNCTION 8 (closure)
12 STORE_FAST 1 (foo)
%3d 14 LOAD_FAST 1 (foo)
16 RETURN_VALUE
""" % (_h.__code__.co_firstlineno + 1,
__file__,
_h.__code__.co_firstlineno + 1,
@ -444,15 +446,17 @@ dis_nested_0 = """\
dis_nested_1 = """%s
Disassembly of <code object foo at 0x..., file "%s", line %d>:
%3d 0 LOAD_CLOSURE 1 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>')
8 MAKE_FUNCTION 8 (closure)
10 LOAD_DEREF 2 (y)
12 GET_ITER
14 CALL_FUNCTION 1
16 RETURN_VALUE
0 MAKE_CELL 1 (x)
%3d 2 LOAD_CLOSURE 1 (x)
4 BUILD_TUPLE 1
6 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
8 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>')
10 MAKE_FUNCTION 8 (closure)
12 LOAD_DEREF 2 (y)
14 GET_ITER
16 CALL_FUNCTION 1
18 RETURN_VALUE
""" % (dis_nested_0,
__file__,
_h.__code__.co_firstlineno + 1,
@ -958,59 +962,64 @@ expected_jumpy_line = 1
#print('expected_opinfo_jumpy = [\n ',
#',\n '.join(map(str, _instructions)), ',\n]', sep='')
#dis.dis(outer)
Instruction = dis.Instruction
expected_opinfo_outer = [
Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='a', argrepr='a', offset=0, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='b', argrepr='b', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=4, starts_line=2, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=7, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=40, starts_line=8, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=42, starts_line=None, is_jump_target=False),
]
expected_opinfo_f = [
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=5, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=6, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=6, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=34, starts_line=6, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='c', argrepr='c', offset=0, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='d', argrepr='d', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=4, starts_line=3, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=5, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=6, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='c', argrepr='c', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=24, starts_line=5, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='a', argrepr='a', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=6, argval='b', argrepr='b', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='c', argrepr='c', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='d', argrepr='d', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=38, starts_line=6, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=40, starts_line=None, is_jump_target=False),
]
expected_opinfo_inner = [
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='', offset=14, starts_line=None, is_jump_target=False),

51
Lib/test/test_scope.py
View File

@ -176,6 +176,57 @@ class ScopeTests(unittest.TestCase):
self.assertEqual(foo(a=42), 50)
self.assertEqual(foo(), 25)
def testCellIsArgAndEscapes(self):
# We need to be sure that a cell passed in as an arg still
# gets wrapped in a new cell if the arg escapes into an
# inner function (closure).
def external():
value = 42
def inner():
return value
cell, = inner.__closure__
return cell
cell_ext = external()
def spam(arg):
def eggs():
return arg
return eggs
eggs = spam(cell_ext)
cell_closure, = eggs.__closure__
cell_eggs = eggs()
self.assertIs(cell_eggs, cell_ext)
self.assertIsNot(cell_eggs, cell_closure)
def testCellIsLocalAndEscapes(self):
# We need to be sure that a cell bound to a local still
# gets wrapped in a new cell if the local escapes into an
# inner function (closure).
def external():
value = 42
def inner():
return value
cell, = inner.__closure__
return cell
cell_ext = external()
def spam(arg):
cell = arg
def eggs():
return cell
return eggs
eggs = spam(cell_ext)
cell_closure, = eggs.__closure__
cell_eggs = eggs()
self.assertIs(cell_eggs, cell_ext)
self.assertIsNot(cell_eggs, cell_closure)
def testRecursion(self):
def f(x):

5
Misc/NEWS.d/next/Core and Builtins/2021-05-26-19-10-47.bpo-43693.1KSG9u.rst Normal file
View File

@ -0,0 +1,5 @@
A new opcode MAKE_CELL has been added that effectively moves some of
the work done on function entry into the compiler and into the eval
loop. In addition to creating the required cell objects, the new
opcode converts relevant arguments (and other locals) to cell
variables on function entry.

131
Objects/frameobject.c
View File

@ -918,6 +918,19 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
return f;
}
int
_PyFrame_OpAlreadyRan(PyFrameObject *f, int opcode, int oparg)
{
const _Py_CODEUNIT *code =
(const _Py_CODEUNIT *)PyBytes_AS_STRING(f->f_code->co_code);
for (int i = 0; i < f->f_lasti; i++) {
if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) {
return 1;
}
}
return 0;
}
int
PyFrame_FastToLocalsWithError(PyFrameObject *f)
{
@ -961,15 +974,52 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
former here and will later use the cell for the variable.
*/
if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
assert(fast[i] == NULL);
continue;
}
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
PyObject *value = fast[i];
if (kind & (CO_FAST_CELL | CO_FAST_FREE) && value != NULL) {
assert(PyCell_Check(value));
value = PyCell_GET(value);
if (f->f_state != FRAME_CLEARED) {
int cellargoffset = CO_CELL_NOT_AN_ARG;
if (co->co_cell2arg != NULL) {
cellargoffset = co->co_cell2arg[i - co->co_nlocals];
}
if (kind & CO_FAST_FREE) {
// The cell was set by _PyEval_MakeFrameVector() from
// the function's closure.
assert(value != NULL && PyCell_Check(value));
value = PyCell_GET(value);
}
else if (kind & CO_FAST_CELL) {
// Note that no *_DEREF ops can happen before MAKE_CELL
// executes. So there's no need to duplicate the work
// that MAKE_CELL would otherwise do later, if it hasn't
// run yet.
if (value != NULL) {
if (PyCell_Check(value) &&
_PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) {
// (likely) MAKE_CELL must have executed already.
value = PyCell_GET(value);
}
// (unlikely) Otherwise it must be an initial value set
// by an earlier call to PyFrame_FastToLocals().
}
else {
// (unlikely) MAKE_CELL hasn't executed yet.
if (cellargoffset != CO_CELL_NOT_AN_ARG) {
// It is an arg that escapes into an inner
// function so we use the initial value that
// was already set by _PyEval_MakeFrameVector().
// Normally the arg value would always be set.
// However, it can be NULL if it was deleted via
// PyFrame_LocalsToFast().
value = fast[cellargoffset];
}
}
}
}
else {
assert(value == NULL);
}
if (value == NULL) {
if (PyObject_DelItem(locals, name) != 0) {
@ -1010,8 +1060,9 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
PyObject **fast;
PyObject *error_type, *error_value, *error_traceback;
PyCodeObject *co;
if (f == NULL)
if (f == NULL || f->f_state == FRAME_CLEARED) {
return;
}
locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
if (locals == NULL)
return;
@ -1039,16 +1090,68 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
continue;
}
}
if (kind & (CO_FAST_CELL | CO_FAST_FREE)) {
assert(PyCell_Check(fast[i]));
if (PyCell_GET(fast[i]) != value) {
if (PyCell_Set(fast[i], value) < 0) {
PyErr_Clear();
}
PyObject *oldvalue = fast[i];
int cellargoffset = CO_CELL_NOT_AN_ARG;
if (kind & CO_FAST_CELL && co->co_cell2arg != NULL) {
assert(i >= co->co_nlocals);
cellargoffset = co->co_cell2arg[i - co->co_nlocals];
}
PyObject *cell = NULL;
if (kind == CO_FAST_FREE) {
// The cell was set by _PyEval_MakeFrameVector() from
// the function's closure.
assert(oldvalue != NULL && PyCell_Check(oldvalue));
cell = oldvalue;
}
else if (kind & CO_FAST_CELL && oldvalue != NULL) {
if (cellargoffset != CO_CELL_NOT_AN_ARG) {
// (likely) MAKE_CELL must have executed already.
// It's the cell for an arg.
assert(PyCell_Check(oldvalue));
cell = oldvalue;
}
else {
if (PyCell_Check(oldvalue) &&
_PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) {
// (likely) MAKE_CELL must have executed already.
cell = oldvalue;
}
// (unlikely) Otherwise, it must have been set to some
// initial value by an earlier call to PyFrame_LocalsToFast().
}
}
if (cell != NULL) {
oldvalue = PyCell_GET(cell);
if (value != oldvalue) {
Py_XDECREF(oldvalue);
Py_XINCREF(value);
PyCell_SET(cell, value);
}
}
else {
int offset = i;
if (kind & CO_FAST_CELL) {
// (unlikely) MAKE_CELL hasn't executed yet.
// Note that there is no need to create the cell that
// MAKE_CELL would otherwise create later, since no
// *_DEREF ops can happen before MAKE_CELL has run.
if (cellargoffset != CO_CELL_NOT_AN_ARG) {
// It's the cell for an arg.
// Replace the initial value that was set by
// _PyEval_MakeFrameVector().
// Normally the arg value would always be set.
// However, it can be NULL if it was deleted
// via an earlier PyFrame_LocalsToFast() call.
offset = cellargoffset;
oldvalue = fast[offset];
}
// Otherwise set an initial value for MAKE_CELL to use
// when it runs later.
}
if (value != oldvalue) {
Py_XINCREF(value);
Py_XSETREF(fast[offset], value);
}
} else if (fast[i] != value) {
Py_XINCREF(value);
Py_XSETREF(fast[i], value);
}
Py_XDECREF(value);
}

13
Objects/typeobject.c
View File

@ -11,6 +11,8 @@
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_unionobject.h" // _Py_Union(), _Py_union_type_or
#include "frameobject.h"
#include "pycore_frame.h" // _PyFrame_OpAlreadyRan
#include "opcode.h" // MAKE_CELL
#include "structmember.h" // PyMemberDef
#include <ctype.h>
@ -8877,14 +8879,17 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
}
PyObject *obj = f->f_localsptr[0];
Py_ssize_t i;
int i;
if (obj == NULL && co->co_cell2arg) {
/* The first argument might be a cell. */
for (i = 0; i < co->co_ncellvars; i++) {
if (co->co_cell2arg[i] == 0) {
PyObject *cell = f->f_localsptr[co->co_nlocals + i];
assert(PyCell_Check(cell));
obj = PyCell_GET(cell);
int celloffset = co->co_nlocals + i;
PyObject *cell = f->f_localsptr[celloffset];
if (PyCell_Check(cell) &&
_PyFrame_OpAlreadyRan(f, MAKE_CELL, celloffset)) {
obj = PyCell_GET(cell);
}
break;
}
}

49
Python/ceval.c
View File

@ -3076,6 +3076,34 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
goto error;
}
case TARGET(MAKE_CELL): {
PyObject *initial = GETLOCAL(oparg);
// Normally initial would be NULL. However, it
// might have been set to an initial value during
// a call to PyFrame_LocalsToFast().
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
goto error;
}
/* If it is an arg then copy the arg into the cell. */
if (initial == NULL && co->co_cell2arg != NULL) {
int argoffset = co->co_cell2arg[oparg - co->co_nlocals];
if (argoffset != CO_CELL_NOT_AN_ARG) {
PyObject *arg = GETLOCAL(argoffset);
// It will have been set in initialize_locals() but
// may have been deleted PyFrame_LocalsToFast().
if (arg != NULL) {;
Py_INCREF(arg);
PyCell_SET(cell, arg);
/* Clear the local copy. */
SETLOCAL(argoffset, NULL);
}
}
}
SETLOCAL(oparg, cell);
DISPATCH();
}
case TARGET(DELETE_DEREF): {
PyObject *cell = GETLOCAL(oparg);
PyObject *oldobj = PyCell_GET(cell);
@ -5067,27 +5095,6 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
}
}
/* Allocate and initialize storage for cell vars, and copy free
vars into frame. */
for (i = 0; i < co->co_ncellvars; ++i) {
PyObject *c;
Py_ssize_t arg;
/* Possibly account for the cell variable being an argument. */
if (co->co_cell2arg != NULL &&
(arg = co->co_cell2arg[i]) != CO_CELL_NOT_AN_ARG) {
c = PyCell_New(GETLOCAL(arg));
/* Clear the local copy. */
SETLOCAL(arg, NULL);
}
else {
c = PyCell_New(NULL);
}
if (c == NULL)
goto fail;
SETLOCAL(co->co_nlocals + i, c);
}
/* Copy closure variables to free variables */
for (i = 0; i < co->co_nfreevars; ++i) {
PyObject *o = PyTuple_GET_ITEM(con->fc_closure, i);

72
Python/compile.c
View File

@ -1185,6 +1185,8 @@ stack_effect(int opcode, int oparg, int jump)
return -1;
/* Closures */
case MAKE_CELL:
return 0;
case LOAD_CLOSURE:
return 1;
case LOAD_DEREF:
@ -7374,15 +7376,47 @@ optimize_cfg(struct compiler *c, struct assembler *a, PyObject *consts);
static int
ensure_exits_have_lineno(struct compiler *c);
static inline int
insert_instruction(basicblock *block, int pos, struct instr *instr) {
if (compiler_next_instr(block) < 0) {
return -1;
}
for (int i = block->b_iused-1; i > pos; i--) {
block->b_instr[i] = block->b_instr[i-1];
}
block->b_instr[pos] = *instr;
return 0;
}
static int
insert_generator_prefix(struct compiler *c, basicblock *entryblock) {
insert_prefix_instructions(struct compiler *c, basicblock *entryblock) {
int flags = compute_code_flags(c);
if (flags < 0) {
return -1;
}
int kind;
/* Set up cells for any variable that escapes, to be put in a closure. */
PyObject *k, *v;
Py_ssize_t pos = 0;
while (PyDict_Next(c->u->u_cellvars, &pos, &k, &v)) {
assert(PyLong_AS_LONG(v) < INT_MAX);
int cellindex = (int)PyLong_AS_LONG(v);
struct instr make_cell = {
.i_opcode = MAKE_CELL,
// This will get fixed in offset_derefs().
.i_oparg = cellindex,
.i_lineno = -1,
.i_target = NULL,
};
if (insert_instruction(entryblock, (int)(pos - 1), &make_cell) < 0) {
return -1;
}
}
/* Add the generator prefix instructions. */
if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
int kind;
if (flags & CO_COROUTINE) {
kind = 1;
}
@ -7392,20 +7426,18 @@ insert_generator_prefix(struct compiler *c, basicblock *entryblock) {
else {
kind = 0;
}
struct instr gen_start = {
.i_opcode = GEN_START,
.i_oparg = kind,
.i_lineno = -1,
.i_target = NULL,
};
if (insert_instruction(entryblock, 0, &gen_start) < 0) {
return -1;
}
}
else {
return 0;
}
if (compiler_next_instr(entryblock) < 0) {
return -1;
}
for (int i = entryblock->b_iused-1; i > 0; i--) {
entryblock->b_instr[i] = entryblock->b_instr[i-1];
}
entryblock->b_instr[0].i_opcode = GEN_START;
entryblock->b_instr[0].i_oparg = kind;
entryblock->b_instr[0].i_lineno = -1;
entryblock->b_instr[0].i_target = NULL;
return 0;
}
@ -7438,12 +7470,15 @@ guarantee_lineno_for_exits(struct assembler *a, int firstlineno) {
}
static void
offset_derefs(basicblock *entryblock, int nlocals)
fix_cell_offsets(struct compiler *c, basicblock *entryblock)
{
assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX);
int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames);
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
for (int i = 0; i < b->b_iused; i++) {
struct instr *inst = &b->b_instr[i];
switch(inst->i_opcode) {
case MAKE_CELL:
case LOAD_CLOSURE:
case LOAD_DEREF:
case STORE_DEREF:
@ -7493,7 +7528,7 @@ assemble(struct compiler *c, int addNone)
}
assert(entryblock != NULL);
if (insert_generator_prefix(c, entryblock)) {
if (insert_prefix_instructions(c, entryblock)) {
goto error;
}
@ -7510,8 +7545,7 @@ assemble(struct compiler *c, int addNone)
a.a_entry = entryblock;
a.a_nblocks = nblocks;
assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX);
offset_derefs(entryblock, (int)PyDict_GET_SIZE(c->u->u_varnames));
fix_cell_offsets(c, entryblock);
consts = consts_dict_keys_inorder(c->u->u_consts);
if (consts == NULL) {

2940
Python/importlib.h generated
View File

File diff suppressed because it is too large Load Diff

5269
Python/importlib_external.h generated
View File

File diff suppressed because it is too large Load Diff

2
Python/opcode_targets.h generated
View File

@ -134,12 +134,12 @@ static void *opcode_targets[256] = {
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
&&_unknown_opcode,
&&TARGET_MAKE_CELL,
&&TARGET_LOAD_CLOSURE,
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_CALL_FUNCTION_KW,
&&TARGET_CALL_FUNCTION_EX,
&&_unknown_opcode,