From 13d52c268699f199a8e917a0f1dc4c51e5346c42 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 10 Mar 2020 18:52:34 +0200 Subject: [PATCH] bpo-34822: Simplify AST for subscription. (GH-9605) * Remove the slice type. * Make Slice a kind of the expr type instead of the slice type. * Replace ExtSlice(slices) with Tuple(slices, Load()). * Replace Index(value) with a value itself. All non-terminal nodes in AST for expressions are now of the expr type. --- Doc/library/ast.rst | 68 ++- Doc/tools/susp-ignored.csv | 2 + Doc/whatsnew/3.9.rst | 13 + Include/Python-ast.h | 41 +- Lib/ast.py | 33 +- Lib/test/test_ast.py | 24 +- Lib/test/test_type_comments.py | 2 +- .../2018-09-27-19-31-47.bpo-34822.EztBhL.rst | 5 + Parser/Python.asdl | 9 +- Python/Python-ast.c | 393 +++++------------- Python/ast.c | 88 +--- Python/ast_opt.c | 37 +- Python/ast_unparse.c | 51 +-- Python/compile.c | 210 +++------- Python/symtable.c | 33 +- 15 files changed, 300 insertions(+), 709 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-09-27-19-31-47.bpo-34822.EztBhL.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a11f8b98940..45f16c17d3d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -120,13 +120,26 @@ Node classes Class :class:`ast.Constant` is now used for all constants. +.. versionchanged:: 3.9 + + Simple indices are represented by their value, extended slices are + represented as tuples. + .. deprecated:: 3.8 Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meanwhile, + but they will be removed in future Python releases. In the meantime, instantiating them will return an instance of a different class. +.. deprecated:: 3.9 + + Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + available, but they will be removed in future Python releases. + In the meantime, instantiating them will return an instance of + a different class. + + Literals ^^^^^^^^ @@ -552,30 +565,33 @@ Subscripting .. class:: Subscript(value, slice, ctx) - A subscript, such as ``l[1]``. ``value`` is the object, often a - :class:`Name`. ``slice`` is one of :class:`Index`, :class:`Slice` or - :class:`ExtSlice`. ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` + A subscript, such as ``l[1]``. ``value`` is the subscripted object + (usually sequence or mapping). ``slice`` is an index, slice or key. + It can be a :class:`Tuple` and contain a :class:`Slice`. + ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to the action performed with the subscript. - -.. class:: Index(value) - - Simple subscripting with a single value - .. doctest:: - >>> print(ast.dump(ast.parse('l[1]', mode='eval'), indent=4)) + >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) Expression( body=Subscript( value=Name(id='l', ctx=Load()), - slice=Index( - value=Constant(value=1)), + slice=Tuple( + elts=[ + Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + Constant(value=3)], + ctx=Load()), ctx=Load())) .. class:: Slice(lower, upper, step) - Regular slicing (on the form x:y). + Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). + Can occur only inside the *slice* field of :class:`Subscript`, either + directly or as an element of :class:`Tuple`. .. doctest:: @@ -589,27 +605,6 @@ Subscripting ctx=Load())) -.. class:: ExtSlice(dims) - - Advanced slicing. ``dims`` holds a list of :class:`Slice` and - :class:`Index` nodes - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=ExtSlice( - dims=[ - Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - Index( - value=Constant(value=3))]), - ctx=Load())) - - Comprehensions ~~~~~~~~~~~~~~ @@ -823,8 +818,7 @@ Statements AnnAssign( target=Subscript( value=Name(id='a', ctx=Load()), - slice=Index( - value=Constant(value=1)), + slice=Constant(value=1), ctx=Store()), annotation=Name(id='int', ctx=Load()), simple=0)], @@ -1708,7 +1702,7 @@ and classes for traversing abstract syntax trees: def visit_Name(self, node): return Subscript( value=Name(id='data', ctx=Load()), - slice=Index(value=Constant(value=node.id)), + slice=Constant(value=node.id), ctx=node.ctx ), node) diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index d3901be2f56..7be8d0abd69 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -108,6 +108,8 @@ howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) howto/urllib2,,:password,"""joe:password@example.com""" +library/ast,,:upper,lower:upper +library/ast,,:step,lower:upper:step library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 90562893dae..690dbba2c97 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -535,6 +535,12 @@ Deprecated (Contributed by Victor Stinner in :issue:`39353`.) +* :mod:`ast` classes ``Index`` and ``ExtSlice`` are considered deprecated + and will be removed in future Python versions. ``value`` itself should be + used instead of ``Index(value)``. ``Tuple(slices, Load())`` should be + used instead of ``ExtSlice(slices)``. + (Contributed by Serhiy Storchaka in :issue:`32892`.) + * The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized @@ -667,10 +673,17 @@ Changes in the Python API since the *buffering* parameter has been removed. (Contributed by Victor Stinner in :issue:`39357`.) +* Simplified AST for subscription. Simple indices will be represented by + their value, extended slices will be represented as tuples. + ``Index(value)`` will return a ``value`` itself, ``ExtSlice(slices)`` + will return ``Tuple(slices, Load())``. + (Contributed by Serhiy Storchaka in :issue:`34822`.) + * The :mod:`importlib` module now ignores the :envvar:`PYTHONCASEOK` environment variable when the :option:`-E` or :option:`-I` command line options are being used. + CPython bytecode changes ------------------------ diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 931a6b945b4..f4631f2f9b5 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -20,8 +20,6 @@ typedef struct _expr *expr_ty; typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5, Param=6 } expr_context_ty; -typedef struct _slice *slice_ty; - typedef enum _boolop { And=1, Or=2 } boolop_ty; typedef enum _operator { Add=1, Sub=2, Mult=3, MatMult=4, Div=5, Mod=6, Pow=7, @@ -233,7 +231,7 @@ enum _expr_kind {BoolOp_kind=1, NamedExpr_kind=2, BinOp_kind=3, UnaryOp_kind=4, YieldFrom_kind=15, Compare_kind=16, Call_kind=17, FormattedValue_kind=18, JoinedStr_kind=19, Constant_kind=20, Attribute_kind=21, Subscript_kind=22, Starred_kind=23, - Name_kind=24, List_kind=25, Tuple_kind=26}; + Name_kind=24, List_kind=25, Tuple_kind=26, Slice_kind=27}; struct _expr { enum _expr_kind kind; union { @@ -346,7 +344,7 @@ struct _expr { struct { expr_ty value; - slice_ty slice; + expr_ty slice; expr_context_ty ctx; } Subscript; @@ -370,32 +368,17 @@ struct _expr { expr_context_ty ctx; } Tuple; - } v; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3}; -struct _slice { - enum _slice_kind kind; - union { struct { expr_ty lower; expr_ty upper; expr_ty step; } Slice; - struct { - asdl_seq *dims; - } ExtSlice; - - struct { - expr_ty value; - } Index; - } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; }; struct _comprehension { @@ -648,7 +631,7 @@ expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int +expr_ty _Py_Subscript(expr_ty value, expr_ty slice, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6) @@ -667,12 +650,10 @@ expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3) -slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); -#define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1) -slice_ty _Py_ExtSlice(asdl_seq * dims, PyArena *arena); -#define Index(a0, a1) _Py_Index(a0, a1) -slice_ty _Py_Index(expr_ty value, PyArena *arena); +#define Slice(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Slice(a0, a1, a2, a3, a4, a5, a6, a7) +expr_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4) comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async, PyArena *arena); diff --git a/Lib/ast.py b/Lib/ast.py index 0bce4a49dc7..8b88d0fac05 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -445,7 +445,7 @@ class NodeTransformer(NodeVisitor): def visit_Name(self, node): return copy_location(Subscript( value=Name(id='data', ctx=Load()), - slice=Index(value=Str(s=node.id)), + slice=Constant(value=node.id), ctx=node.ctx ), node) @@ -552,6 +552,7 @@ _const_types = { _const_types_not = { Num: (bool,), } + _const_node_type_names = { bool: 'NameConstant', # should be before int type(None): 'NameConstant', @@ -563,6 +564,23 @@ _const_node_type_names = { type(...): 'Ellipsis', } +class Index(AST): + def __new__(cls, value, **kwargs): + return value + +class ExtSlice(AST): + def __new__(cls, dims=(), **kwargs): + return Tuple(list(dims), Load(), **kwargs) + +def _dims_getter(self): + return self.elts + +def _dims_setter(self, value): + self.elts = value + +Tuple.dims = property(_dims_getter, _dims_setter) + + # Large float and imaginary literals get turned into infinities in the AST. # We unparse those infinities to INFSTR. _INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) @@ -1268,10 +1286,8 @@ class _Unparser(NodeVisitor): self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) with self.delimit("[", "]"): - if (isinstance(node.slice, Index) - and isinstance(node.slice.value, Tuple) - and node.slice.value.elts): - self.items_view(self.traverse, node.slice.value.elts) + if isinstance(node.slice, Tuple) and node.slice.elts: + self.items_view(self.traverse, node.slice.elts) else: self.traverse(node.slice) @@ -1283,10 +1299,6 @@ class _Unparser(NodeVisitor): def visit_Ellipsis(self, node): self.write("...") - def visit_Index(self, node): - self.set_precedence(_Precedence.TUPLE, node.value) - self.traverse(node.value) - def visit_Slice(self, node): if node.lower: self.traverse(node.lower) @@ -1297,9 +1309,6 @@ class _Unparser(NodeVisitor): self.write(":") self.traverse(node.step) - def visit_ExtSlice(self, node): - self.items_view(self.traverse, node.dims) - def visit_arg(self, node): self.write(node.arg) if node.annotation: diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index c1e9f002811..0058e932f6a 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -343,6 +343,10 @@ class AST_Tests(unittest.TestCase): def test_field_attr_existence(self): for name, item in ast.__dict__.items(): if self._is_ast_node(name, item): + if name == 'Index': + # Index(value) just returns value now. + # The argument is required. + continue x = item() if isinstance(x, ast.AST): self.assertEqual(type(x._fields), tuple) @@ -1308,11 +1312,11 @@ class ASTValidatorTests(unittest.TestCase): self.expr(attr, "must have Load context") def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)), + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3), ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())), + sub = ast.Subscript(x, ast.Name("y", ast.Store()), ast.Load()) self.expr(sub, "must have Load context") s = ast.Name("x", ast.Store()) @@ -1320,9 +1324,9 @@ class ASTValidatorTests(unittest.TestCase): sl = ast.Slice(*args) self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - sl = ast.ExtSlice([]) - self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice") - sl = ast.ExtSlice([ast.Index(s)]) + sl = ast.Tuple([], ast.Load()) + self.expr(ast.Subscript(x, sl, ast.Load())) + sl = ast.Tuple([s], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") def test_starred(self): @@ -1664,11 +1668,11 @@ class EndPositionTests(unittest.TestCase): ''').strip() i1, i2, im = map(self._parse_value, (s1, s2, sm)) self._check_content(s1, i1.value, 'f()[1, 2]') - self._check_content(s1, i1.value.slice.value, '1, 2') + self._check_content(s1, i1.value.slice, '1, 2') self._check_content(s2, i2.slice.lower, 'a.b') self._check_content(s2, i2.slice.upper, 'c.d') - self._check_content(sm, im.slice.dims[0].upper, 'f ()') - self._check_content(sm, im.slice.dims[1].lower, 'g ()') + self._check_content(sm, im.slice.elts[0].upper, 'f ()') + self._check_content(sm, im.slice.elts[1].lower, 'g ()') self._check_end_pos(im, 3, 3) def test_binop(self): @@ -1989,13 +1993,13 @@ eval_results = [ ('Expression', ('Constant', (1, 0, 1, 2), 10, None)), ('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), ('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), ('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), ('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), ('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), ('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), ('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), ('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), ] main() diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 43be54efdbd..017073a9f1d 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -390,7 +390,7 @@ class TypeCommentTests(unittest.TestCase): arg = tree.argtypes[0] self.assertEqual(arg.id, "int") self.assertEqual(tree.returns.value.id, "List") - self.assertEqual(tree.returns.slice.value.id, "str") + self.assertEqual(tree.returns.slice.id, "str") tree = parse_func_type_input("(int, *str, **Any) -> float") self.assertEqual(tree.argtypes[0].id, "int") diff --git a/Misc/NEWS.d/next/Library/2018-09-27-19-31-47.bpo-34822.EztBhL.rst b/Misc/NEWS.d/next/Library/2018-09-27-19-31-47.bpo-34822.EztBhL.rst new file mode 100644 index 00000000000..96cbfb2d9ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-09-27-19-31-47.bpo-34822.EztBhL.rst @@ -0,0 +1,5 @@ +Simplified AST for subscription. Simple indices are now represented by their +value, extended slices are represented as tuples. :mod:`ast` classes +``Index`` and ``ExtSlice`` are considered deprecated and will be removed in +future Python versions. In the meantime, ``Index(value)`` now returns +a ``value`` itself, ``ExtSlice(slices)`` returns ``Tuple(slices, Load())``. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index bec30a7a1f3..a1ddebdbcc7 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -78,21 +78,20 @@ module Python -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) + | Subscript(expr value, expr slice, expr_context ctx) | Starred(expr value, expr_context ctx) | Name(identifier id, expr_context ctx) | List(expr* elts, expr_context ctx) | Tuple(expr* elts, expr_context ctx) + -- can appear only in Subscript + | Slice(expr? lower, expr? upper, expr? step) + -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) expr_context = Load | Store | Del | AugLoad | AugStore | Param - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - boolop = And | Or operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 47c88b67417..f58dd9c2178 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -52,7 +52,6 @@ typedef struct { PyObject *ExceptHandler_type; PyObject *Expr_type; PyObject *Expression_type; - PyObject *ExtSlice_type; PyObject *FloorDiv_singleton; PyObject *FloorDiv_type; PyObject *For_type; @@ -71,7 +70,6 @@ typedef struct { PyObject *Import_type; PyObject *In_singleton; PyObject *In_type; - PyObject *Index_type; PyObject *Interactive_type; PyObject *Invert_singleton; PyObject *Invert_type; @@ -166,7 +164,6 @@ typedef struct { PyObject *ctx; PyObject *decorator_list; PyObject *defaults; - PyObject *dims; PyObject *elt; PyObject *elts; PyObject *end_col_offset; @@ -213,7 +210,6 @@ typedef struct { PyObject *right; PyObject *simple; PyObject *slice; - PyObject *slice_type; PyObject *step; PyObject *stmt_type; PyObject *tag; @@ -281,7 +277,6 @@ static int astmodule_clear(PyObject *module) Py_CLEAR(astmodulestate(module)->ExceptHandler_type); Py_CLEAR(astmodulestate(module)->Expr_type); Py_CLEAR(astmodulestate(module)->Expression_type); - Py_CLEAR(astmodulestate(module)->ExtSlice_type); Py_CLEAR(astmodulestate(module)->FloorDiv_singleton); Py_CLEAR(astmodulestate(module)->FloorDiv_type); Py_CLEAR(astmodulestate(module)->For_type); @@ -300,7 +295,6 @@ static int astmodule_clear(PyObject *module) Py_CLEAR(astmodulestate(module)->Import_type); Py_CLEAR(astmodulestate(module)->In_singleton); Py_CLEAR(astmodulestate(module)->In_type); - Py_CLEAR(astmodulestate(module)->Index_type); Py_CLEAR(astmodulestate(module)->Interactive_type); Py_CLEAR(astmodulestate(module)->Invert_singleton); Py_CLEAR(astmodulestate(module)->Invert_type); @@ -395,7 +389,6 @@ static int astmodule_clear(PyObject *module) Py_CLEAR(astmodulestate(module)->ctx); Py_CLEAR(astmodulestate(module)->decorator_list); Py_CLEAR(astmodulestate(module)->defaults); - Py_CLEAR(astmodulestate(module)->dims); Py_CLEAR(astmodulestate(module)->elt); Py_CLEAR(astmodulestate(module)->elts); Py_CLEAR(astmodulestate(module)->end_col_offset); @@ -442,7 +435,6 @@ static int astmodule_clear(PyObject *module) Py_CLEAR(astmodulestate(module)->right); Py_CLEAR(astmodulestate(module)->simple); Py_CLEAR(astmodulestate(module)->slice); - Py_CLEAR(astmodulestate(module)->slice_type); Py_CLEAR(astmodulestate(module)->step); Py_CLEAR(astmodulestate(module)->stmt_type); Py_CLEAR(astmodulestate(module)->tag); @@ -509,7 +501,6 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) Py_VISIT(astmodulestate(module)->ExceptHandler_type); Py_VISIT(astmodulestate(module)->Expr_type); Py_VISIT(astmodulestate(module)->Expression_type); - Py_VISIT(astmodulestate(module)->ExtSlice_type); Py_VISIT(astmodulestate(module)->FloorDiv_singleton); Py_VISIT(astmodulestate(module)->FloorDiv_type); Py_VISIT(astmodulestate(module)->For_type); @@ -528,7 +519,6 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) Py_VISIT(astmodulestate(module)->Import_type); Py_VISIT(astmodulestate(module)->In_singleton); Py_VISIT(astmodulestate(module)->In_type); - Py_VISIT(astmodulestate(module)->Index_type); Py_VISIT(astmodulestate(module)->Interactive_type); Py_VISIT(astmodulestate(module)->Invert_singleton); Py_VISIT(astmodulestate(module)->Invert_type); @@ -623,7 +613,6 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) Py_VISIT(astmodulestate(module)->ctx); Py_VISIT(astmodulestate(module)->decorator_list); Py_VISIT(astmodulestate(module)->defaults); - Py_VISIT(astmodulestate(module)->dims); Py_VISIT(astmodulestate(module)->elt); Py_VISIT(astmodulestate(module)->elts); Py_VISIT(astmodulestate(module)->end_col_offset); @@ -670,7 +659,6 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) Py_VISIT(astmodulestate(module)->right); Py_VISIT(astmodulestate(module)->simple); Py_VISIT(astmodulestate(module)->slice); - Py_VISIT(astmodulestate(module)->slice_type); Py_VISIT(astmodulestate(module)->step); Py_VISIT(astmodulestate(module)->stmt_type); Py_VISIT(astmodulestate(module)->tag); @@ -733,7 +721,6 @@ static int init_identifiers(void) if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return 0; if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return 0; if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return 0; - if ((state->dims = PyUnicode_InternFromString("dims")) == NULL) return 0; if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return 0; if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return 0; if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0; @@ -1035,19 +1022,12 @@ static const char * const Tuple_fields[]={ "elts", "ctx", }; -static PyObject* ast2obj_expr_context(expr_context_ty); -static PyObject* ast2obj_slice(void*); static const char * const Slice_fields[]={ "lower", "upper", "step", }; -static const char * const ExtSlice_fields[]={ - "dims", -}; -static const char * const Index_fields[]={ - "value", -}; +static PyObject* ast2obj_expr_context(expr_context_ty); static PyObject* ast2obj_boolop(boolop_ty); static PyObject* ast2obj_operator(operator_ty); static PyObject* ast2obj_unaryop(unaryop_ty); @@ -1635,6 +1615,14 @@ static int init_types(void) if (!state->List_type) return 0; state->Tuple_type = make_type("Tuple", state->expr_type, Tuple_fields, 2); if (!state->Tuple_type) return 0; + state->Slice_type = make_type("Slice", state->expr_type, Slice_fields, 3); + if (!state->Slice_type) return 0; + if (PyObject_SetAttr(state->Slice_type, state->lower, Py_None) == -1) + return 0; + if (PyObject_SetAttr(state->Slice_type, state->upper, Py_None) == -1) + return 0; + if (PyObject_SetAttr(state->Slice_type, state->step, Py_None) == -1) + return 0; state->expr_context_type = make_type("expr_context", state->AST_type, NULL, 0); if (!state->expr_context_type) return 0; @@ -1673,22 +1661,6 @@ static int init_types(void) state->Param_singleton = PyType_GenericNew((PyTypeObject *)state->Param_type, NULL, NULL); if (!state->Param_singleton) return 0; - state->slice_type = make_type("slice", state->AST_type, NULL, 0); - if (!state->slice_type) return 0; - if (!add_attributes(state->slice_type, NULL, 0)) return 0; - state->Slice_type = make_type("Slice", state->slice_type, Slice_fields, 3); - if (!state->Slice_type) return 0; - if (PyObject_SetAttr(state->Slice_type, state->lower, Py_None) == -1) - return 0; - if (PyObject_SetAttr(state->Slice_type, state->upper, Py_None) == -1) - return 0; - if (PyObject_SetAttr(state->Slice_type, state->step, Py_None) == -1) - return 0; - state->ExtSlice_type = make_type("ExtSlice", state->slice_type, - ExtSlice_fields, 1); - if (!state->ExtSlice_type) return 0; - state->Index_type = make_type("Index", state->slice_type, Index_fields, 1); - if (!state->Index_type) return 0; state->boolop_type = make_type("boolop", state->AST_type, NULL, 0); if (!state->boolop_type) return 0; if (!add_attributes(state->boolop_type, NULL, 0)) return 0; @@ -1929,7 +1901,6 @@ static int obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena); static int obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena); static int obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena); -static int obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena); static int obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena); static int obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena); static int obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena); @@ -3092,7 +3063,7 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int } expr_ty -Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int +Subscript(expr_ty value, expr_ty slice, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; @@ -3227,46 +3198,22 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int return p; } -slice_ty -Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena) +expr_ty +Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena) { - slice_ty p; - p = (slice_ty)PyArena_Malloc(arena, sizeof(*p)); + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Slice_kind; p->v.Slice.lower = lower; p->v.Slice.upper = upper; p->v.Slice.step = step; - return p; -} - -slice_ty -ExtSlice(asdl_seq * dims, PyArena *arena) -{ - slice_ty p; - p = (slice_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = ExtSlice_kind; - p->v.ExtSlice.dims = dims; - return p; -} - -slice_ty -Index(expr_ty value, PyArena *arena) -{ - slice_ty p; - if (!value) { - PyErr_SetString(PyExc_ValueError, - "field value is required for Index"); - return NULL; - } - p = (slice_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Index_kind; - p->v.Index.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -4389,7 +4336,7 @@ ast2obj_expr(void* _o) if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_slice(o->v.Subscript.slice); + value = ast2obj_expr(o->v.Subscript.slice); if (!value) goto failed; if (PyObject_SetAttr(result, astmodulestate_global->slice, value) == -1) goto failed; @@ -4460,6 +4407,26 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case Slice_kind: + tp = (PyTypeObject *)astmodulestate_global->Slice_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.Slice.lower); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->lower, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.Slice.upper); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->upper, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.Slice.step); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->step, value) == -1) + goto failed; + Py_DECREF(value); + break; } value = ast2obj_int(o->lineno); if (!value) goto failed; @@ -4516,65 +4483,6 @@ PyObject* ast2obj_expr_context(expr_context_ty o) return NULL; } } -PyObject* -ast2obj_slice(void* _o) -{ - slice_ty o = (slice_ty)_o; - PyObject *result = NULL, *value = NULL; - PyTypeObject *tp; - if (!o) { - Py_RETURN_NONE; - } - - switch (o->kind) { - case Slice_kind: - tp = (PyTypeObject *)astmodulestate_global->Slice_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - value = ast2obj_expr(o->v.Slice.lower); - if (!value) goto failed; - if (PyObject_SetAttr(result, astmodulestate_global->lower, value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.Slice.upper); - if (!value) goto failed; - if (PyObject_SetAttr(result, astmodulestate_global->upper, value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.Slice.step); - if (!value) goto failed; - if (PyObject_SetAttr(result, astmodulestate_global->step, value) == -1) - goto failed; - Py_DECREF(value); - break; - case ExtSlice_kind: - tp = (PyTypeObject *)astmodulestate_global->ExtSlice_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice); - if (!value) goto failed; - if (PyObject_SetAttr(result, astmodulestate_global->dims, value) == -1) - goto failed; - Py_DECREF(value); - break; - case Index_kind: - tp = (PyTypeObject *)astmodulestate_global->Index_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - value = ast2obj_expr(o->v.Index.value); - if (!value) goto failed; - if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1) - goto failed; - Py_DECREF(value); - break; - } - return result; -failed: - Py_XDECREF(value); - Py_XDECREF(result); - return NULL; -} - PyObject* ast2obj_boolop(boolop_ty o) { switch(o) { @@ -8421,7 +8329,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } if (isinstance) { expr_ty value; - slice_ty slice; + expr_ty slice; expr_context_ty ctx; if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) { @@ -8446,7 +8354,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } else { int res; - res = obj2ast_slice(tmp, &slice, arena); + res = obj2ast_expr(tmp, &slice, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } @@ -8668,6 +8576,60 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + tp = astmodulestate_global->Slice_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty lower; + expr_ty upper; + expr_ty step; + + if (_PyObject_LookupAttr(obj, astmodulestate_global->lower, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + lower = NULL; + } + else { + int res; + res = obj2ast_expr(tmp, &lower, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, astmodulestate_global->upper, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + upper = NULL; + } + else { + int res; + res = obj2ast_expr(tmp, &upper, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, astmodulestate_global->step, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + step = NULL; + } + else { + int res; + res = obj2ast_expr(tmp, &step, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = Slice(lower, upper, step, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %R", obj); failed: @@ -8733,148 +8695,6 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena) return 1; } -int -obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) -{ - int isinstance; - - PyObject *tmp = NULL; - PyObject *tp; - - if (obj == Py_None) { - *out = NULL; - return 0; - } - tp = astmodulestate_global->Slice_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - expr_ty lower; - expr_ty upper; - expr_ty step; - - if (_PyObject_LookupAttr(obj, astmodulestate_global->lower, &tmp) < 0) { - return 1; - } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - lower = NULL; - } - else { - int res; - res = obj2ast_expr(tmp, &lower, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - if (_PyObject_LookupAttr(obj, astmodulestate_global->upper, &tmp) < 0) { - return 1; - } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - upper = NULL; - } - else { - int res; - res = obj2ast_expr(tmp, &upper, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - if (_PyObject_LookupAttr(obj, astmodulestate_global->step, &tmp) < 0) { - return 1; - } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - step = NULL; - } - else { - int res; - res = obj2ast_expr(tmp, &step, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - *out = Slice(lower, upper, step, arena); - if (*out == NULL) goto failed; - return 0; - } - tp = astmodulestate_global->ExtSlice_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - asdl_seq* dims; - - if (_PyObject_LookupAttr(obj, astmodulestate_global->dims, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"dims\" missing from ExtSlice"); - return 1; - } - else { - int res; - Py_ssize_t len; - Py_ssize_t i; - if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "ExtSlice field \"dims\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); - goto failed; - } - len = PyList_GET_SIZE(tmp); - dims = _Py_asdl_seq_new(len, arena); - if (dims == NULL) goto failed; - for (i = 0; i < len; i++) { - slice_ty val; - PyObject *tmp2 = PyList_GET_ITEM(tmp, i); - Py_INCREF(tmp2); - res = obj2ast_slice(tmp2, &val, arena); - Py_DECREF(tmp2); - if (res != 0) goto failed; - if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "ExtSlice field \"dims\" changed size during iteration"); - goto failed; - } - asdl_seq_SET(dims, i, val); - } - Py_CLEAR(tmp); - } - *out = ExtSlice(dims, arena); - if (*out == NULL) goto failed; - return 0; - } - tp = astmodulestate_global->Index_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - expr_ty value; - - if (_PyObject_LookupAttr(obj, astmodulestate_global->value, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Index"); - return 1; - } - else { - int res; - res = obj2ast_expr(tmp, &value, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - *out = Index(value, arena); - if (*out == NULL) goto failed; - return 0; - } - - PyErr_Format(PyExc_TypeError, "expected some sort of slice, but got %R", obj); - failed: - Py_XDECREF(tmp); - return 1; -} - int obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena) { @@ -10187,6 +10007,10 @@ PyInit__ast(void) goto error; } Py_INCREF(astmodulestate(m)->Tuple_type); + if (PyModule_AddObject(m, "Slice", astmodulestate_global->Slice_type) < 0) { + goto error; + } + Py_INCREF(astmodulestate(m)->Slice_type); if (PyModule_AddObject(m, "expr_context", astmodulestate_global->expr_context_type) < 0) { goto error; @@ -10218,23 +10042,6 @@ PyInit__ast(void) goto error; } Py_INCREF(astmodulestate(m)->Param_type); - if (PyModule_AddObject(m, "slice", astmodulestate_global->slice_type) < 0) { - goto error; - } - Py_INCREF(astmodulestate(m)->slice_type); - if (PyModule_AddObject(m, "Slice", astmodulestate_global->Slice_type) < 0) { - goto error; - } - Py_INCREF(astmodulestate(m)->Slice_type); - if (PyModule_AddObject(m, "ExtSlice", astmodulestate_global->ExtSlice_type) - < 0) { - goto error; - } - Py_INCREF(astmodulestate(m)->ExtSlice_type); - if (PyModule_AddObject(m, "Index", astmodulestate_global->Index_type) < 0) { - goto error; - } - Py_INCREF(astmodulestate(m)->Index_type); if (PyModule_AddObject(m, "boolop", astmodulestate_global->boolop_type) < 0) { goto error; diff --git a/Python/ast.c b/Python/ast.c index 43b50c5dd4c..62ee60aa9ff 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -39,31 +39,6 @@ validate_comprehension(asdl_seq *gens) return 1; } -static int -validate_slice(slice_ty slice) -{ - switch (slice->kind) { - case Slice_kind: - return (!slice->v.Slice.lower || validate_expr(slice->v.Slice.lower, Load)) && - (!slice->v.Slice.upper || validate_expr(slice->v.Slice.upper, Load)) && - (!slice->v.Slice.step || validate_expr(slice->v.Slice.step, Load)); - case ExtSlice_kind: { - Py_ssize_t i; - if (!validate_nonempty_seq(slice->v.ExtSlice.dims, "dims", "ExtSlice")) - return 0; - for (i = 0; i < asdl_seq_LEN(slice->v.ExtSlice.dims); i++) - if (!validate_slice(asdl_seq_GET(slice->v.ExtSlice.dims, i))) - return 0; - return 1; - } - case Index_kind: - return validate_expr(slice->v.Index.value, Load); - default: - PyErr_SetString(PyExc_SystemError, "unknown slice node"); - return 0; - } -} - static int validate_keywords(asdl_seq *keywords) { @@ -309,10 +284,14 @@ validate_expr(expr_ty exp, expr_context_ty ctx) case Attribute_kind: return validate_expr(exp->v.Attribute.value, Load); case Subscript_kind: - return validate_slice(exp->v.Subscript.slice) && + return validate_expr(exp->v.Subscript.slice, Load) && validate_expr(exp->v.Subscript.value, Load); case Starred_kind: return validate_expr(exp->v.Starred.value, ctx); + case Slice_kind: + return (!exp->v.Slice.lower || validate_expr(exp->v.Slice.lower, Load)) && + (!exp->v.Slice.upper || validate_expr(exp->v.Slice.upper, Load)) && + (!exp->v.Slice.step || validate_expr(exp->v.Slice.step, Load)); case List_kind: return validate_exprs(exp->v.List.elts, ctx, 0); case Tuple_kind: @@ -2471,7 +2450,7 @@ ast_for_atom(struct compiling *c, const node *n) } } -static slice_ty +static expr_ty ast_for_slice(struct compiling *c, const node *n) { node *ch; @@ -2485,13 +2464,7 @@ ast_for_slice(struct compiling *c, const node *n) */ ch = CHILD(n, 0); if (NCH(n) == 1 && TYPE(ch) == test) { - /* 'step' variable hold no significance in terms of being used over - other vars */ - step = ast_for_expr(c, ch); - if (!step) - return NULL; - - return Index(step, c->c_arena); + return ast_for_expr(c, ch); } if (TYPE(ch) == test) { @@ -2533,7 +2506,8 @@ ast_for_slice(struct compiling *c, const node *n) } } - return Slice(lower, upper, step, c->c_arena); + return Slice(lower, upper, step, LINENO(n), n->n_col_offset, + n->n_end_lineno, n->n_end_col_offset, c->c_arena); } static expr_ty @@ -2621,7 +2595,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr, const nod REQ(CHILD(n, 2), RSQB); n = CHILD(n, 1); if (NCH(n) == 1) { - slice_ty slc = ast_for_slice(c, CHILD(n, 0)); + expr_ty slc = ast_for_slice(c, CHILD(n, 0)); if (!slc) return NULL; return Subscript(left_expr, slc, Load, LINENO(start), start->n_col_offset, @@ -2629,47 +2603,27 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr, const nod c->c_arena); } else { - /* The grammar is ambiguous here. The ambiguity is resolved - by treating the sequence as a tuple literal if there are - no slice features. - */ - Py_ssize_t j; - slice_ty slc; - expr_ty e; - int simple = 1; - asdl_seq *slices, *elts; - slices = _Py_asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); - if (!slices) + int j; + expr_ty slc, e; + asdl_seq *elts; + elts = _Py_asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); + if (!elts) return NULL; for (j = 0; j < NCH(n); j += 2) { slc = ast_for_slice(c, CHILD(n, j)); if (!slc) return NULL; - if (slc->kind != Index_kind) - simple = 0; - asdl_seq_SET(slices, j / 2, slc); - } - if (!simple) { - return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(start), start->n_col_offset, - n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); - } - /* extract Index values and put them in a Tuple */ - elts = _Py_asdl_seq_new(asdl_seq_LEN(slices), c->c_arena); - if (!elts) - return NULL; - for (j = 0; j < asdl_seq_LEN(slices); ++j) { - slc = (slice_ty)asdl_seq_GET(slices, j); - assert(slc->kind == Index_kind && slc->v.Index.value); - asdl_seq_SET(elts, j, slc->v.Index.value); + asdl_seq_SET(elts, j / 2, slc); } e = Tuple(elts, Load, LINENO(n), n->n_col_offset, - n->n_end_lineno, n->n_end_col_offset, c->c_arena); + n->n_end_lineno, n->n_end_col_offset, + c->c_arena); if (!e) return NULL; - return Subscript(left_expr, Index(e, c->c_arena), + return Subscript(left_expr, e, Load, LINENO(start), start->n_col_offset, - n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); + n_copy->n_end_lineno, n_copy->n_end_col_offset, + c->c_arena); } } } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 39e164adb8c..7a2b6e6aab1 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -310,20 +310,16 @@ fold_subscr(expr_ty node, PyArena *arena, int optimize) { PyObject *newval; expr_ty arg, idx; - slice_ty slice; arg = node->v.Subscript.value; - slice = node->v.Subscript.slice; + idx = node->v.Subscript.slice; if (node->v.Subscript.ctx != Load || arg->kind != Constant_kind || - /* TODO: handle other types of slices */ - slice->kind != Index_kind || - slice->v.Index.value->kind != Constant_kind) + idx->kind != Constant_kind) { return 1; } - idx = slice->v.Index.value; newval = PyObject_GetItem(arg->v.Constant.value, idx->v.Constant.value); return make_const(node, newval, arena); } @@ -395,7 +391,6 @@ static int astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_); static int astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_); static int astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_); static int astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_); -static int astfold_slice(slice_ty node_, PyArena *ctx_, int optimize_); static int astfold_arg(arg_ty node_, PyArena *ctx_, int optimize_); static int astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_); static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_); @@ -548,12 +543,17 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_) break; case Subscript_kind: CALL(astfold_expr, expr_ty, node_->v.Subscript.value); - CALL(astfold_slice, slice_ty, node_->v.Subscript.slice); + CALL(astfold_expr, expr_ty, node_->v.Subscript.slice); CALL(fold_subscr, expr_ty, node_); break; case Starred_kind: CALL(astfold_expr, expr_ty, node_->v.Starred.value); break; + case Slice_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.lower); + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.upper); + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.step); + break; case List_kind: CALL_SEQ(astfold_expr, expr_ty, node_->v.List.elts); break; @@ -572,27 +572,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_) return 1; } -static int -astfold_slice(slice_ty node_, PyArena *ctx_, int optimize_) -{ - switch (node_->kind) { - case Slice_kind: - CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.lower); - CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.upper); - CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.step); - break; - case ExtSlice_kind: - CALL_SEQ(astfold_slice, slice_ty, node_->v.ExtSlice.dims); - break; - case Index_kind: - CALL(astfold_expr, expr_ty, node_->v.Index.value); - break; - default: - break; - } - return 1; -} - static int astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_) { diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index bd9c1396c07..5ecd1b0fef5 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -17,7 +17,7 @@ append_joinedstr(_PyUnicodeWriter *writer, expr_ty e, bool is_format_spec); static int append_formattedvalue(_PyUnicodeWriter *writer, expr_ty e); static int -append_ast_slice(_PyUnicodeWriter *writer, slice_ty slice); +append_ast_slice(_PyUnicodeWriter *writer, expr_ty e); static int append_charp(_PyUnicodeWriter *writer, const char *charp) @@ -718,62 +718,31 @@ append_ast_attribute(_PyUnicodeWriter *writer, expr_ty e) } static int -append_ast_simple_slice(_PyUnicodeWriter *writer, slice_ty slice) +append_ast_slice(_PyUnicodeWriter *writer, expr_ty e) { - if (slice->v.Slice.lower) { - APPEND_EXPR(slice->v.Slice.lower, PR_TEST); + if (e->v.Slice.lower) { + APPEND_EXPR(e->v.Slice.lower, PR_TEST); } APPEND_STR(":"); - if (slice->v.Slice.upper) { - APPEND_EXPR(slice->v.Slice.upper, PR_TEST); + if (e->v.Slice.upper) { + APPEND_EXPR(e->v.Slice.upper, PR_TEST); } - if (slice->v.Slice.step) { + if (e->v.Slice.step) { APPEND_STR(":"); - APPEND_EXPR(slice->v.Slice.step, PR_TEST); + APPEND_EXPR(e->v.Slice.step, PR_TEST); } return 0; } -static int -append_ast_ext_slice(_PyUnicodeWriter *writer, slice_ty slice) -{ - Py_ssize_t i, dims_count; - dims_count = asdl_seq_LEN(slice->v.ExtSlice.dims); - for (i = 0; i < dims_count; i++) { - APPEND_STR_IF(i > 0, ", "); - APPEND(slice, (slice_ty)asdl_seq_GET(slice->v.ExtSlice.dims, i)); - } - APPEND_STR_IF(dims_count == 1, ","); - return 0; -} - -static int -append_ast_slice(_PyUnicodeWriter *writer, slice_ty slice) -{ - switch (slice->kind) { - case Slice_kind: - return append_ast_simple_slice(writer, slice); - case ExtSlice_kind: - return append_ast_ext_slice(writer, slice); - case Index_kind: - APPEND_EXPR(slice->v.Index.value, PR_TUPLE); - return 0; - default: - PyErr_SetString(PyExc_SystemError, - "unexpected slice kind"); - return -1; - } -} - static int append_ast_subscript(_PyUnicodeWriter *writer, expr_ty e) { APPEND_EXPR(e->v.Subscript.value, PR_ATOM); APPEND_STR("["); - APPEND(slice, e->v.Subscript.slice); + APPEND_EXPR(e->v.Subscript.slice, PR_TUPLE); APPEND_STR_FINISH("]"); } @@ -878,6 +847,8 @@ append_ast_expr(_PyUnicodeWriter *writer, expr_ty e, int level) return append_ast_subscript(writer, e); case Starred_kind: return append_ast_starred(writer, e); + case Slice_kind: + return append_ast_slice(writer, e); case Name_kind: return _PyUnicodeWriter_WriteStr(writer, e->v.Name.id); case List_kind: diff --git a/Python/compile.c b/Python/compile.c index f228e16079b..55333b39d3e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -193,8 +193,8 @@ static int compiler_visit_keyword(struct compiler *, keyword_ty); static int compiler_visit_expr(struct compiler *, expr_ty); static int compiler_augassign(struct compiler *, stmt_ty); static int compiler_annassign(struct compiler *, stmt_ty); -static int compiler_visit_slice(struct compiler *, slice_ty, - expr_context_ty); +static int compiler_subscript(struct compiler *, expr_ty); +static int compiler_slice(struct compiler *, expr_ty); static int inplace_binop(struct compiler *, operator_ty); static int are_all_items_const(asdl_seq *, Py_ssize_t, Py_ssize_t); @@ -4045,14 +4045,11 @@ check_subscripter(struct compiler *c, expr_ty e) } static int -check_index(struct compiler *c, expr_ty e, slice_ty s) +check_index(struct compiler *c, expr_ty e, expr_ty s) { PyObject *v; - if (s->kind != Index_kind) { - return 1; - } - PyTypeObject *index_type = infer_type(s->v.Index.value); + PyTypeObject *index_type = infer_type(s); if (index_type == NULL || PyType_FastSubclass(index_type, Py_TPFLAGS_LONG_SUBCLASS) || index_type == &PySlice_Type) { @@ -5065,39 +5062,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) } break; case Subscript_kind: - switch (e->v.Subscript.ctx) { - case AugLoad: - VISIT(c, expr, e->v.Subscript.value); - VISIT_SLICE(c, e->v.Subscript.slice, AugLoad); - break; - case Load: - if (!check_subscripter(c, e->v.Subscript.value)) { - return 0; - } - if (!check_index(c, e->v.Subscript.value, e->v.Subscript.slice)) { - return 0; - } - VISIT(c, expr, e->v.Subscript.value); - VISIT_SLICE(c, e->v.Subscript.slice, Load); - break; - case AugStore: - VISIT_SLICE(c, e->v.Subscript.slice, AugStore); - break; - case Store: - VISIT(c, expr, e->v.Subscript.value); - VISIT_SLICE(c, e->v.Subscript.slice, Store); - break; - case Del: - VISIT(c, expr, e->v.Subscript.value); - VISIT_SLICE(c, e->v.Subscript.slice, Del); - break; - case Param: - default: - PyErr_SetString(PyExc_SystemError, - "param invalid in subscript expression"); - return 0; - } - break; + return compiler_subscript(c, e); case Starred_kind: switch (e->v.Starred.ctx) { case Store: @@ -5109,6 +5074,9 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) return compiler_error(c, "can't use starred expression here"); } + break; + case Slice_kind: + return compiler_slice(c, e); case Name_kind: return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ @@ -5213,68 +5181,35 @@ check_annotation(struct compiler *c, stmt_ty s) } static int -check_ann_slice(struct compiler *c, slice_ty sl) -{ - switch(sl->kind) { - case Index_kind: - return check_ann_expr(c, sl->v.Index.value); - case Slice_kind: - if (sl->v.Slice.lower && !check_ann_expr(c, sl->v.Slice.lower)) { - return 0; - } - if (sl->v.Slice.upper && !check_ann_expr(c, sl->v.Slice.upper)) { - return 0; - } - if (sl->v.Slice.step && !check_ann_expr(c, sl->v.Slice.step)) { - return 0; - } - break; - default: - PyErr_SetString(PyExc_SystemError, - "unexpected slice kind"); - return 0; - } - return 1; -} - -static int -check_ann_subscr(struct compiler *c, slice_ty sl) +check_ann_subscr(struct compiler *c, expr_ty e) { /* We check that everything in a subscript is defined at runtime. */ - Py_ssize_t i, n; - - switch (sl->kind) { - case Index_kind: + switch (e->kind) { case Slice_kind: - if (!check_ann_slice(c, sl)) { + if (e->v.Slice.lower && !check_ann_expr(c, e->v.Slice.lower)) { return 0; } - break; - case ExtSlice_kind: - n = asdl_seq_LEN(sl->v.ExtSlice.dims); + if (e->v.Slice.upper && !check_ann_expr(c, e->v.Slice.upper)) { + return 0; + } + if (e->v.Slice.step && !check_ann_expr(c, e->v.Slice.step)) { + return 0; + } + return 1; + case Tuple_kind: { + /* extended slice */ + asdl_seq *elts = e->v.Tuple.elts; + Py_ssize_t i, n = asdl_seq_LEN(elts); for (i = 0; i < n; i++) { - slice_ty subsl = (slice_ty)asdl_seq_GET(sl->v.ExtSlice.dims, i); - switch (subsl->kind) { - case Index_kind: - case Slice_kind: - if (!check_ann_slice(c, subsl)) { - return 0; - } - break; - case ExtSlice_kind: - default: - PyErr_SetString(PyExc_SystemError, - "extended slice invalid in nested slice"); + if (!check_ann_subscr(c, asdl_seq_GET(elts, i))) { return 0; } } - break; - default: - PyErr_Format(PyExc_SystemError, - "invalid subscript kind %d", sl->kind); - return 0; + return 1; + } + default: + return check_ann_expr(c, e); } - return 1; } static int @@ -5400,12 +5335,20 @@ compiler_warn(struct compiler *c, const char *format, ...) } static int -compiler_handle_subscr(struct compiler *c, const char *kind, - expr_context_ty ctx) +compiler_subscript(struct compiler *c, expr_ty e) { + expr_context_ty ctx = e->v.Subscript.ctx; int op = 0; - /* XXX this code is duplicated */ + if (ctx == Load) { + if (!check_subscripter(c, e->v.Subscript.value)) { + return 0; + } + if (!check_index(c, e->v.Subscript.value, e->v.Subscript.slice)) { + return 0; + } + } + switch (ctx) { case AugLoad: /* fall through to Load */ case Load: op = BINARY_SUBSCR; break; @@ -5413,23 +5356,26 @@ compiler_handle_subscr(struct compiler *c, const char *kind, case Store: op = STORE_SUBSCR; break; case Del: op = DELETE_SUBSCR; break; case Param: - PyErr_Format(PyExc_SystemError, - "invalid %s kind %d in subscript\n", - kind, ctx); + PyErr_SetString(PyExc_SystemError, + "param invalid in subscript expression"); return 0; } - if (ctx == AugLoad) { - ADDOP(c, DUP_TOP_TWO); - } - else if (ctx == AugStore) { + if (ctx == AugStore) { ADDOP(c, ROT_THREE); } + else { + VISIT(c, expr, e->v.Subscript.value); + VISIT(c, expr, e->v.Subscript.slice); + if (ctx == AugLoad) { + ADDOP(c, DUP_TOP_TWO); + } + } ADDOP(c, op); return 1; } static int -compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) +compiler_slice(struct compiler *c, expr_ty s) { int n = 2; assert(s->kind == Slice_kind); @@ -5457,64 +5403,6 @@ compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) return 1; } -static int -compiler_visit_nested_slice(struct compiler *c, slice_ty s, - expr_context_ty ctx) -{ - switch (s->kind) { - case Slice_kind: - return compiler_slice(c, s, ctx); - case Index_kind: - VISIT(c, expr, s->v.Index.value); - break; - case ExtSlice_kind: - default: - PyErr_SetString(PyExc_SystemError, - "extended slice invalid in nested slice"); - return 0; - } - return 1; -} - -static int -compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) -{ - const char * kindname = NULL; - switch (s->kind) { - case Index_kind: - kindname = "index"; - if (ctx != AugStore) { - VISIT(c, expr, s->v.Index.value); - } - break; - case Slice_kind: - kindname = "slice"; - if (ctx != AugStore) { - if (!compiler_slice(c, s, ctx)) - return 0; - } - break; - case ExtSlice_kind: - kindname = "extended slice"; - if (ctx != AugStore) { - Py_ssize_t i, n = asdl_seq_LEN(s->v.ExtSlice.dims); - for (i = 0; i < n; i++) { - slice_ty sub = (slice_ty)asdl_seq_GET( - s->v.ExtSlice.dims, i); - if (!compiler_visit_nested_slice(c, sub, ctx)) - return 0; - } - ADDOP_I(c, BUILD_TUPLE, n); - } - break; - default: - PyErr_Format(PyExc_SystemError, - "invalid subscript kind %d", s->kind); - return 0; - } - return compiler_handle_subscr(c, kindname, ctx); -} - /* End of the compiler section, beginning of the assembler section */ /* do depth-first search of basic block graph, starting with block. diff --git a/Python/symtable.c b/Python/symtable.c index 290e41b64ac..014570e6ef7 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -202,7 +202,6 @@ static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); -static int symtable_visit_slice(struct symtable *st, slice_ty); static int symtable_visit_params(struct symtable *st, asdl_seq *args); static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); @@ -1632,11 +1631,19 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Subscript_kind: VISIT(st, expr, e->v.Subscript.value); - VISIT(st, slice, e->v.Subscript.slice); + VISIT(st, expr, e->v.Subscript.slice); break; case Starred_kind: VISIT(st, expr, e->v.Starred.value); break; + case Slice_kind: + if (e->v.Slice.lower) + VISIT(st, expr, e->v.Slice.lower) + if (e->v.Slice.upper) + VISIT(st, expr, e->v.Slice.upper) + if (e->v.Slice.step) + VISIT(st, expr, e->v.Slice.step) + break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) @@ -1841,28 +1848,6 @@ symtable_visit_keyword(struct symtable *st, keyword_ty k) } -static int -symtable_visit_slice(struct symtable *st, slice_ty s) -{ - switch (s->kind) { - case Slice_kind: - if (s->v.Slice.lower) - VISIT(st, expr, s->v.Slice.lower) - if (s->v.Slice.upper) - VISIT(st, expr, s->v.Slice.upper) - if (s->v.Slice.step) - VISIT(st, expr, s->v.Slice.step) - break; - case ExtSlice_kind: - VISIT_SEQ(st, slice, s->v.ExtSlice.dims) - break; - case Index_kind: - VISIT(st, expr, s->v.Index.value) - break; - } - return 1; -} - static int symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_seq *generators,