traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-39320: Handle unpacking of *values in compiler (GH-17984) 

* Add three new bytecodes: LIST_TO_TUPLE, LIST_EXTEND, SET_UPDATE. Use them to implement star unpacking expressions.

* Remove four bytecodes BUILD_LIST_UNPACK, BUILD_TUPLE_UNPACK, BUILD_SET_UNPACK and  BUILD_TUPLE_UNPACK_WITH_CALL opcodes as they are now unused.

* Update magic number and dis.rst for new bytecodes.
This commit is contained in:
Mark Shannon 2020-01-23 09:25:17 +00:00 committed by GitHub
parent f9e07e116c
commit 13bc13960c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 2349 additions and 2360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Doc/library/dis.rst
View File

@ -859,40 +859,25 @@ All of the following opcodes use their arguments.
.. versionadded:: 3.6
.. opcode:: BUILD_TUPLE_UNPACK (count)
.. opcode:: LIST_TO_TUPLE
Pops *count* iterables from the stack, joins them in a single tuple,
and pushes the result. Implements iterable unpacking in tuple
displays ``(*x, *y, *z)``.
Pops a list from the stack and pushes a tuple containing the same values.
.. versionadded:: 3.5
.. versionadded:: 3.9
.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count)
.. opcode:: LIST_EXTEND (i)
This is similar to :opcode:`BUILD_TUPLE_UNPACK`,
but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position
``count + 1`` should be the corresponding callable ``f``.
Calls ``list.extend(TOS1[-i], TOS)``. Used to build lists.
.. versionadded:: 3.6
.. versionadded:: 3.9
.. opcode:: BUILD_LIST_UNPACK (count)
.. opcode:: SET_UPDATE
This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list
instead of tuple. Implements iterable unpacking in list
displays ``[*x, *y, *z]``.
Calls ``set.update(TOS1[-i], TOS)``. Used to build sets.
.. versionadded:: 3.5
.. opcode:: BUILD_SET_UNPACK (count)
This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set
instead of tuple. Implements iterable unpacking in set
displays ``{*x, *y, *z}``.
.. versionadded:: 3.5
.. versionadded:: 3.9
.. opcode:: BUILD_MAP_UNPACK (count)
@ -1124,10 +1109,6 @@ All of the following opcodes use their arguments.
Calls a callable object with variable set of positional and keyword
arguments. If the lowest bit of *flags* is set, the top of the stack
contains a mapping object containing additional keyword arguments.
Below that is an iterable object containing positional arguments and
a callable object to call. :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and
:opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` can be used for merging multiple
mapping objects and iterables containing arguments.
Before the callable is called, the mapping object and iterable object
are each "unpacked" and their contents passed in as keyword and
positional arguments respectively.

7
Include/opcode.h generated
View File

@ -60,6 +60,7 @@ extern "C" {
#define INPLACE_AND 77
#define INPLACE_XOR 78
#define INPLACE_OR 79
#define LIST_TO_TUPLE 82
#define RETURN_VALUE 83
#define IMPORT_STAR 84
#define SETUP_ANNOTATIONS 85
@ -116,18 +117,16 @@ extern "C" {
#define SET_ADD 146
#define MAP_ADD 147
#define LOAD_CLASSDEREF 148
#define BUILD_LIST_UNPACK 149
#define BUILD_MAP_UNPACK 150
#define BUILD_MAP_UNPACK_WITH_CALL 151
#define BUILD_TUPLE_UNPACK 152
#define BUILD_SET_UNPACK 153
#define SETUP_ASYNC_WITH 154
#define FORMAT_VALUE 155
#define BUILD_CONST_KEY_MAP 156
#define BUILD_STRING 157
#define BUILD_TUPLE_UNPACK_WITH_CALL 158
#define LOAD_METHOD 160
#define CALL_METHOD 161
#define LIST_EXTEND 162
#define SET_UPDATE 163
/* EXCEPT_HANDLER is a special, implicit block type which is created when
entering an except handler. It is not an opcode but we define it here

4
Lib/importlib/_bootstrap_external.py
View File

@ -275,6 +275,8 @@ _code_type = type(_write_atomic.__code__)
# Python 3.9a0 3421 (simplified bytecode for with blocks #32949)
# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
# Python 3.9a2 3424 (simplify bytecodes for *value unpacking)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
# longer be understood by older implementations of the eval loop (usually
@ -283,7 +285,7 @@ _code_type = type(_write_atomic.__code__)
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
MAGIC_NUMBER = (3423).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3424).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'

8
Lib/opcode.py
View File

@ -117,6 +117,7 @@ def_op('INPLACE_AND', 77)
def_op('INPLACE_XOR', 78)
def_op('INPLACE_OR', 79)
def_op('LIST_TO_TUPLE', 82)
def_op('RETURN_VALUE', 83)
def_op('IMPORT_STAR', 84)
def_op('SETUP_ANNOTATIONS', 85)
@ -199,20 +200,19 @@ hasfree.append(148)
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
def_op('BUILD_LIST_UNPACK', 149)
def_op('BUILD_MAP_UNPACK', 150)
def_op('BUILD_MAP_UNPACK_WITH_CALL', 151)
def_op('BUILD_TUPLE_UNPACK', 152)
def_op('BUILD_SET_UNPACK', 153)
jrel_op('SETUP_ASYNC_WITH', 154)
def_op('FORMAT_VALUE', 155)
def_op('BUILD_CONST_KEY_MAP', 156)
def_op('BUILD_STRING', 157)
def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158)
name_op('LOAD_METHOD', 160)
def_op('CALL_METHOD', 161)
def_op('LIST_EXTEND', 162)
def_op('SET_UPDATE', 163)
del def_op, name_op, jrel_op, jabs_op

4
Lib/test/test_extcall.py
View File

@ -252,12 +252,12 @@ What about willful misconduct?
>>> h(1, *h)
Traceback (most recent call last):
...
TypeError: test.test_extcall.h() argument after * must be an iterable, not function
TypeError: Value after * must be an iterable, not function
>>> h(*[1], *h)
Traceback (most recent call last):
...
TypeError: test.test_extcall.h() argument after * must be an iterable, not function
TypeError: Value after * must be an iterable, not function
>>> dir(*h)
Traceback (most recent call last):

15
Misc/NEWS.d/next/Core and Builtins/2020-01-15-15-33-44.bpo-39320.b4hnJW.rst Normal file
View File

@ -0,0 +1,15 @@
Replace four complex bytecodes for building sequences with three simpler ones.
The following four bytecodes have been removed:
* BUILD_LIST_UNPACK
* BUILD_TUPLE_UNPACK
* BUILD_SET_UNPACK
* BUILD_TUPLE_UNPACK_WITH_CALL
The following three bytecodes have been added:
* LIST_TO_TUPLE
* LIST_EXTEND
* SET_UPDATE

87
Python/ceval.c
View File

@ -2621,46 +2621,46 @@ main_loop:
DISPATCH();
}
case TARGET(BUILD_TUPLE_UNPACK_WITH_CALL):
case TARGET(BUILD_TUPLE_UNPACK):
case TARGET(BUILD_LIST_UNPACK): {
int convert_to_tuple = opcode != BUILD_LIST_UNPACK;
Py_ssize_t i;
PyObject *sum = PyList_New(0);
PyObject *return_value;
if (sum == NULL)
case TARGET(LIST_TO_TUPLE): {
PyObject *list = POP();
PyObject *tuple = PyList_AsTuple(list);
Py_DECREF(list);
if (tuple == NULL) {
goto error;
}
PUSH(tuple);
DISPATCH();
}
for (i = oparg; i > 0; i--) {
PyObject *none_val;
none_val = _PyList_Extend((PyListObject *)sum, PEEK(i));
if (none_val == NULL) {
if (opcode == BUILD_TUPLE_UNPACK_WITH_CALL &&
_PyErr_ExceptionMatches(tstate, PyExc_TypeError))
{
check_args_iterable(tstate, PEEK(1 + oparg), PEEK(i));
}
Py_DECREF(sum);
goto error;
case TARGET(LIST_EXTEND): {
PyObject *iterable = POP();
PyObject *list = PEEK(oparg);
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
(iterable->ob_type->tp_iter == NULL && !PySequence_Check(iterable)))
{
PyErr_Clear();
_PyErr_Format(tstate, PyExc_TypeError,
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
Py_DECREF(none_val);
Py_DECREF(iterable);
goto error;
}
Py_DECREF(none_val);
Py_DECREF(iterable);
DISPATCH();
}
if (convert_to_tuple) {
return_value = PyList_AsTuple(sum);
Py_DECREF(sum);
if (return_value == NULL)
goto error;
case TARGET(SET_UPDATE): {
PyObject *iterable = POP();
PyObject *set = PEEK(oparg);
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
if (err < 0) {
goto error;
}
else {
return_value = sum;
}
while (oparg--)
Py_DECREF(POP());
PUSH(return_value);
DISPATCH();
}
@ -2685,25 +2685,6 @@ main_loop:
DISPATCH();
}
case TARGET(BUILD_SET_UNPACK): {
Py_ssize_t i;
PyObject *sum = PySet_New(NULL);
if (sum == NULL)
goto error;
for (i = oparg; i > 0; i--) {
if (_PySet_Update(sum, PEEK(i)) < 0) {
Py_DECREF(sum);
goto error;
}
}
while (oparg--)
Py_DECREF(POP());
PUSH(sum);
DISPATCH();
}
case TARGET(BUILD_MAP): {
Py_ssize_t i;
PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);

211
Python/compile.c
View File

@ -1007,10 +1007,6 @@ stack_effect(int opcode, int oparg, int jump)
case BUILD_SET:
case BUILD_STRING:
return 1-oparg;
case BUILD_LIST_UNPACK:
case BUILD_TUPLE_UNPACK:
case BUILD_TUPLE_UNPACK_WITH_CALL:
case BUILD_SET_UNPACK:
case BUILD_MAP_UNPACK:
case BUILD_MAP_UNPACK_WITH_CALL:
return 1 - oparg;
@ -1125,6 +1121,11 @@ stack_effect(int opcode, int oparg, int jump)
return 1;
case LOAD_ASSERTION_ERROR:
return 1;
case LIST_TO_TUPLE:
return 0;
case LIST_EXTEND:
case SET_UPDATE:
return -1;
default:
return PY_INVALID_STACK_EFFECT;
}
@ -1705,7 +1706,7 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info,
compiler_pop_fblock(c, POP_VALUE, NULL);
}
return 1;
case FINALLY_END:
if (preserve_tos) {
ADDOP(c, ROT_FOUR);
@ -3675,11 +3676,11 @@ compiler_boolop(struct compiler *c, expr_ty e)
}
static int
starunpack_helper(struct compiler *c, asdl_seq *elts,
int single_op, int inner_op, int outer_op)
starunpack_helper(struct compiler *c, asdl_seq *elts, int pushed,
int build, int add, int extend, int tuple)
{
Py_ssize_t n = asdl_seq_LEN(elts);
Py_ssize_t i, nsubitems = 0, nseen = 0;
Py_ssize_t i, seen_star = 0;
if (n > 2 && are_all_items_const(elts, 0, n)) {
PyObject *folded = PyTuple_New(n);
if (folded == NULL) {
@ -3691,41 +3692,63 @@ starunpack_helper(struct compiler *c, asdl_seq *elts,
Py_INCREF(val);
PyTuple_SET_ITEM(folded, i, val);
}
if (outer_op == BUILD_SET_UNPACK) {
Py_SETREF(folded, PyFrozenSet_New(folded));
if (folded == NULL) {
return 0;
if (tuple) {
ADDOP_LOAD_CONST_NEW(c, folded);
} else {
if (add == SET_ADD) {
Py_SETREF(folded, PyFrozenSet_New(folded));
if (folded == NULL) {
return 0;
}
}
ADDOP_I(c, build, pushed);
ADDOP_LOAD_CONST_NEW(c, folded);
ADDOP_I(c, extend, 1);
}
ADDOP_LOAD_CONST_NEW(c, folded);
ADDOP_I(c, outer_op, 1);
return 1;
}
for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
if (elt->kind == Starred_kind) {
if (nseen) {
ADDOP_I(c, inner_op, nseen);
nseen = 0;
nsubitems++;
seen_star = 1;
}
}
if (seen_star) {
seen_star = 0;
for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
if (elt->kind == Starred_kind) {
if (seen_star == 0) {
ADDOP_I(c, build, i+pushed);
seen_star = 1;
}
VISIT(c, expr, elt->v.Starred.value);
ADDOP_I(c, extend, 1);
}
else {
VISIT(c, expr, elt);
if (seen_star) {
ADDOP_I(c, add, 1);
}
}
VISIT(c, expr, elt->v.Starred.value);
nsubitems++;
}
else {
assert(seen_star);
if (tuple) {
ADDOP(c, LIST_TO_TUPLE);
}
}
else {
for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
VISIT(c, expr, elt);
nseen++;
}
if (tuple) {
ADDOP_I(c, BUILD_TUPLE, n+pushed);
} else {
ADDOP_I(c, build, n+pushed);
}
}
if (nsubitems) {
if (nseen) {
ADDOP_I(c, inner_op, nseen);
nsubitems++;
}
ADDOP_I(c, outer_op, nsubitems);
}
else
ADDOP_I(c, single_op, nseen);
return 1;
}
@ -3767,8 +3790,8 @@ compiler_list(struct compiler *c, expr_ty e)
return assignment_helper(c, elts);
}
else if (e->v.List.ctx == Load) {
return starunpack_helper(c, elts,
BUILD_LIST, BUILD_TUPLE, BUILD_LIST_UNPACK);
return starunpack_helper(c, elts, 0, BUILD_LIST,
LIST_APPEND, LIST_EXTEND, 0);
}
else
VISIT_SEQ(c, expr, elts);
@ -3783,8 +3806,8 @@ compiler_tuple(struct compiler *c, expr_ty e)
return assignment_helper(c, elts);
}
else if (e->v.Tuple.ctx == Load) {
return starunpack_helper(c, elts,
BUILD_TUPLE, BUILD_TUPLE, BUILD_TUPLE_UNPACK);
return starunpack_helper(c, elts, 0, BUILD_LIST,
LIST_APPEND, LIST_EXTEND, 1);
}
else
VISIT_SEQ(c, expr, elts);
@ -3794,8 +3817,8 @@ compiler_tuple(struct compiler *c, expr_ty e)
static int
compiler_set(struct compiler *c, expr_ty e)
{
return starunpack_helper(c, e->v.Set.elts, BUILD_SET,
BUILD_SET, BUILD_SET_UNPACK);
return starunpack_helper(c, e->v.Set.elts, 0, BUILD_SET,
SET_ADD, SET_UPDATE, 0);
}
static int
@ -4184,57 +4207,65 @@ compiler_call_helper(struct compiler *c,
asdl_seq *keywords)
{
Py_ssize_t i, nseen, nelts, nkwelts;
int mustdictunpack = 0;
/* the number of tuples and dictionaries on the stack */
Py_ssize_t nsubargs = 0, nsubkwargs = 0;
nelts = asdl_seq_LEN(args);
nkwelts = asdl_seq_LEN(keywords);
for (i = 0; i < nkwelts; i++) {
keyword_ty kw = asdl_seq_GET(keywords, i);
if (kw->arg == NULL) {
mustdictunpack = 1;
break;
}
}
nseen = n; /* the number of positional arguments on the stack */
for (i = 0; i < nelts; i++) {
expr_ty elt = asdl_seq_GET(args, i);
if (elt->kind == Starred_kind) {
/* A star-arg. If we've seen positional arguments,
pack the positional arguments into a tuple. */
if (nseen) {
ADDOP_I(c, BUILD_TUPLE, nseen);
nseen = 0;
nsubargs++;
}
VISIT(c, expr, elt->v.Starred.value);
nsubargs++;
goto ex_call;
}
else {
VISIT(c, expr, elt);
nseen++;
}
for (i = 0; i < nkwelts; i++) {
keyword_ty kw = asdl_seq_GET(keywords, i);
if (kw->arg == NULL) {
goto ex_call;
}
}
/* Same dance again for keyword arguments */
if (nsubargs || mustdictunpack) {
if (nseen) {
/* Pack up any trailing positional arguments. */
ADDOP_I(c, BUILD_TUPLE, nseen);
nsubargs++;
/* No * or ** args, so can use faster calling sequence */
for (i = 0; i < nelts; i++) {
expr_ty elt = asdl_seq_GET(args, i);
assert(elt->kind != Starred_kind);
VISIT(c, expr, elt);
}
if (nkwelts) {
PyObject *names;
VISIT_SEQ(c, keyword, keywords);
names = PyTuple_New(nkwelts);
if (names == NULL) {
return 0;
}
if (nsubargs > 1) {
/* If we ended up with more than one stararg, we need
to concatenate them into a single sequence. */
ADDOP_I(c, BUILD_TUPLE_UNPACK_WITH_CALL, nsubargs);
}
else if (nsubargs == 0) {
ADDOP_I(c, BUILD_TUPLE, 0);
for (i = 0; i < nkwelts; i++) {
keyword_ty kw = asdl_seq_GET(keywords, i);
Py_INCREF(kw->arg);
PyTuple_SET_ITEM(names, i, kw->arg);
}
ADDOP_LOAD_CONST_NEW(c, names);
ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts);
return 1;
}
else {
ADDOP_I(c, CALL_FUNCTION, n + nelts);
return 1;
}
ex_call:
/* Do positional arguments. */
if (n ==0 && nelts == 1 && ((expr_ty)asdl_seq_GET(args, 0))->kind == Starred_kind) {
VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value);
}
else if (starunpack_helper(c, args, n, BUILD_LIST,
LIST_APPEND, LIST_EXTEND, 1) == 0) {
return 0;
}
/* Then keyword arguments */
if (nkwelts) {
/* the number of dictionaries on the stack */
Py_ssize_t nsubkwargs = 0;
nseen = 0; /* the number of keyword arguments on the stack following */
for (i = 0; i < nkwelts; i++) {
keyword_ty kw = asdl_seq_GET(keywords, i);
@ -4263,29 +4294,9 @@ compiler_call_helper(struct compiler *c,
/* Pack it all up */
ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs);
}
ADDOP_I(c, CALL_FUNCTION_EX, nsubkwargs > 0);
return 1;
}
else if (nkwelts) {
PyObject *names;
VISIT_SEQ(c, keyword, keywords);
names = PyTuple_New(nkwelts);
if (names == NULL) {
return 0;
}
for (i = 0; i < nkwelts; i++) {
keyword_ty kw = asdl_seq_GET(keywords, i);
Py_INCREF(kw->arg);
PyTuple_SET_ITEM(names, i, kw->arg);
}
ADDOP_LOAD_CONST_NEW(c, names);
ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts);
return 1;
}
else {
ADDOP_I(c, CALL_FUNCTION, n + nelts);
return 1;
}
ADDOP_I(c, CALL_FUNCTION_EX, nkwelts > 0);
return 1;
}
@ -4860,9 +4871,9 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, WITH, block);
/* End of body; start the cleanup. */
/* For successful outcome:
* call __exit__(None, None, None)
*/
@ -5984,7 +5995,7 @@ makecode(struct compiler *c, struct assembler *a)
goto error;
}
co = PyCode_NewWithPosOnlyArgs(posonlyargcount+posorkeywordargcount,
posonlyargcount, kwonlyargcount, nlocals_int,
posonlyargcount, kwonlyargcount, nlocals_int,
maxdepth, flags, bytecode, consts, names,
varnames, freevars, cellvars, c->c_filename,
c->u->u_name, c->u->u_firstlineno, a->a_lnotab);

4322
Python/importlib_external.h generated
View File

File diff suppressed because it is too large Load Diff

14
Python/opcode_targets.h generated
View File

@ -81,7 +81,7 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_OR,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_SETUP_ANNOTATIONS,
@ -148,21 +148,21 @@ static void *opcode_targets[256] = {
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
&&TARGET_LOAD_CLASSDEREF,
&&TARGET_BUILD_LIST_UNPACK,
&&_unknown_opcode,
&&TARGET_BUILD_MAP_UNPACK,
&&TARGET_BUILD_MAP_UNPACK_WITH_CALL,
&&TARGET_BUILD_TUPLE_UNPACK,
&&TARGET_BUILD_SET_UNPACK,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_SETUP_ASYNC_WITH,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
&&TARGET_BUILD_TUPLE_UNPACK_WITH_CALL,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_LOAD_METHOD,
&&TARGET_CALL_METHOD,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,