From d51e07f989474f6bc82a96546cd2aeebb81faa79 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 13 Sep 2008 17:43:19 +0000 Subject: [PATCH] Merged revisions 66191,66418,66438,66445 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r66191 | benjamin.peterson | 2008-09-03 17:00:52 -0500 (Wed, 03 Sep 2008) | 1 line update the Grammar file after recent syntax changes ........ r66418 | benjamin.peterson | 2008-09-12 18:49:48 -0500 (Fri, 12 Sep 2008) | 1 line a trival fix to get a few more print corner cases #2899 ........ r66438 | benjamin.peterson | 2008-09-12 21:32:30 -0500 (Fri, 12 Sep 2008) | 5 lines add Jack Diederich's fixer for metaclass syntax #2366 my contribution to this was adding a few tests and fixing a few bugs I also reviewed it (Jack is a committer) ........ r66445 | benjamin.peterson | 2008-09-13 10:50:00 -0500 (Sat, 13 Sep 2008) | 1 line add a few more tests concerning int literals and weird spacing ........ --- Lib/lib2to3/Grammar.txt | 4 +- Lib/lib2to3/fixes/fix_metaclass.py | 227 +++++++++++++++++++++ Lib/lib2to3/fixes/fix_print.py | 2 +- Lib/lib2to3/tests/data/py2_test_grammar.py | 40 +++- Lib/lib2to3/tests/data/py3_test_grammar.py | 44 +++- Lib/lib2to3/tests/test_fixers.py | 146 +++++++++++++ 6 files changed, 452 insertions(+), 11 deletions(-) create mode 100644 Lib/lib2to3/fixes/fix_metaclass.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index abf1e1582e1..1f4a50ffbe0 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -138,7 +138,9 @@ dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite -arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) +arglist: (argument ',')* (argument [','] + |'*' test (',' argument)* [',' '**' test] + |'**' test) argument: test [comp_for] | test '=' test # Really [keyword '='] test comp_iter: comp_for | comp_if diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py new file mode 100644 index 00000000000..e291dd7da12 --- /dev/null +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -0,0 +1,227 @@ +"""Fixer for __metaclass__ = X -> (metaclass=X) methods. + + The various forms of classef (inherits nothing, inherits once, inherints + many) don't parse the same in the CST so we look at ALL classes for + a __metaclass__ and if we find one normalize the inherits to all be + an arglist. + + For one-liner classes ('class X: pass') there is no indent/dedent so + we normalize those into having a suite. + + Moving the __metaclass__ into the classdef can also cause the class + body to be empty so there is some special casing for that as well. + + This fixer also tries very hard to keep original indenting and spacing + in all those corner cases. + +""" +# Author: Jack Diederich + +import os + +# Local imports +from .. import fixer_base +from ..pygram import token +from ..fixer_util import Name, syms, Node, Leaf + + +def has_metaclass(parent): + """ we have to check the cls_node without changing it. + There are two possiblities: + 1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta') + 2) clsdef => simple_stmt => expr_stmt => Leaf('__meta') + """ + for node in parent.children: + if node.type == syms.suite: + return has_metaclass(node) + elif node.type == syms.simple_stmt and node.children: + expr_node = node.children[0] + if expr_node.type == syms.expr_stmt and expr_node.children: + leaf_node = expr_node.children[0] + if leaf_node.value == '__metaclass__': + return True + return False + + +def fixup_parse_tree(cls_node): + """ one-line classes don't get a suite in the parse tree so we add + one to normalize the tree + """ + for node in cls_node.children: + if node.type == syms.suite: + # already in the prefered format, do nothing + return + + # !%@#! oneliners have no suite node, we have to fake one up + for i, node in enumerate(cls_node.children): + if node.type == token.COLON: + break + else: + raise ValueError("No class suite and no ':'!") + + # move everything into a suite node + suite = Node(syms.suite, []) + while cls_node.children[i+1:]: + move_node = cls_node.children[i+1] + suite.append_child(move_node.clone()) + move_node.remove() + cls_node.append_child(suite) + node = suite + + +def fixup_simple_stmt(parent, i, stmt_node): + """ if there is a semi-colon all the parts count as part of the same + simple_stmt. We just want the __metaclass__ part so we move + everything efter the semi-colon into its own simple_stmt node + """ + for semi_ind, node in enumerate(stmt_node.children): + if node.type == token.SEMI: # *sigh* + break + else: + return + + node.remove() # kill the semicolon + new_expr = Node(syms.expr_stmt, []) + new_stmt = Node(syms.simple_stmt, [new_expr]) + while stmt_node.children[semi_ind:]: + move_node = stmt_node.children[semi_ind] + new_expr.append_child(move_node.clone()) + move_node.remove() + parent.insert_child(i, new_stmt) + new_leaf1 = new_stmt.children[0].children[0] + old_leaf1 = stmt_node.children[0].children[0] + new_leaf1.set_prefix(old_leaf1.get_prefix()) + + +def remove_trailing_newline(node): + if node.children and node.children[-1].type == token.NEWLINE: + node.children[-1].remove() + + +def find_metas(cls_node): + # find the suite node (Mmm, sweet nodes) + for node in cls_node.children: + if node.type == syms.suite: + break + else: + raise ValueError("No class suite!") + + # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ] + for i, simple_node in list(enumerate(node.children)): + if simple_node.type == syms.simple_stmt and simple_node.children: + expr_node = simple_node.children[0] + if expr_node.type == syms.expr_stmt and expr_node.children: + leaf_node = expr_node.children[0] + if leaf_node.value == '__metaclass__': + fixup_simple_stmt(node, i, simple_node) + remove_trailing_newline(simple_node) + yield (node, i, simple_node) + + +def fixup_indent(suite): + """ If an INDENT is followed by a thing with a prefix then nuke the prefix + Otherwise we get in trouble when removing __metaclass__ at suite start + """ + kids = suite.children[::-1] + # find the first indent + while kids: + node = kids.pop() + if node.type == token.INDENT: + break + + # find the first Leaf + while kids: + node = kids.pop() + if isinstance(node, Leaf) and node.type != token.DEDENT: + if node.prefix: + node.set_prefix('') + return + else: + kids.extend(node.children[::-1]) + + +class FixMetaclass(fixer_base.BaseFix): + + PATTERN = """ + classdef + """ + + def transform(self, node, results): + if not has_metaclass(node): + return node + + fixup_parse_tree(node) + + # find metaclasses, keep the last one + last_metaclass = None + for suite, i, stmt in find_metas(node): + last_metaclass = stmt + stmt.remove() + + text_type = node.children[0].type # always Leaf(nnn, 'class') + + # figure out what kind of classdef we have + if len(node.children) == 7: + # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite]) + # 0 1 2 3 4 5 6 + if node.children[3].type == syms.arglist: + arglist = node.children[3] + # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite]) + elif isinstance(node.children[3], Leaf): + parent = node.children[3].clone() + arglist = Node(syms.arglist, [parent]) + node.set_child(3, arglist) + else: + raise ValueError("Unexpected class inheritance arglist") + elif len(node.children) == 6: + # Node(classdef, ['class', 'name', '(', ')', ':', suite]) + # 0 1 2 3 4 5 + arglist = Node(syms.arglist, []) + node.insert_child(3, arglist) + elif len(node.children) == 4: + # Node(classdef, ['class', 'name', ':', suite]) + # 0 1 2 3 + arglist = Node(syms.arglist, []) + node.insert_child(2, Leaf(token.RPAR, ')')) + node.insert_child(2, arglist) + node.insert_child(2, Leaf(token.LPAR, '(')) + else: + raise ValueError("Unexpected class definition") + + # now stick the metaclass in the arglist + meta_txt = last_metaclass.children[0].children[0] + meta_txt.value = 'metaclass' + orig_meta_prefix = meta_txt.get_prefix() + + if arglist.children: + arglist.append_child(Leaf(token.COMMA, ',')) + meta_txt.set_prefix(' ') + else: + meta_txt.set_prefix('') + + # compact the expression "metaclass = Meta" -> "metaclass=Meta" + expr_stmt = last_metaclass.children[0] + assert expr_stmt.type == syms.expr_stmt + expr_stmt.children[1].set_prefix('') + expr_stmt.children[2].set_prefix('') + + arglist.append_child(last_metaclass) + + fixup_indent(suite) + + # check for empty suite + if not suite.children: + # one-liner that was just __metaclass_ + suite.remove() + pass_leaf = Leaf(text_type, 'pass') + pass_leaf.set_prefix(orig_meta_prefix) + node.append_child(pass_leaf) + node.append_child(Leaf(token.NEWLINE, os.linesep)) + + elif len(suite.children) > 1 and \ + (suite.children[-2].type == token.INDENT and + suite.children[-1].type == token.DEDENT): + # there was only one line in the class body and it was __metaclass__ + pass_leaf = Leaf(text_type, 'pass') + suite.insert_child(-1, pass_leaf) + suite.insert_child(-1, Leaf(token.NEWLINE, os.linesep)) diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py index 6d01dfd931f..134a972b07d 100644 --- a/Lib/lib2to3/fixes/fix_print.py +++ b/Lib/lib2to3/fixes/fix_print.py @@ -29,7 +29,7 @@ parend_expr = patcomp.compile_pattern( class FixPrint(fixer_base.ConditionalFix): PATTERN = """ - simple_stmt< bare='print' any > | print_stmt + simple_stmt< any* bare='print' any* > | print_stmt """ skip_on = '__future__.print_function' diff --git a/Lib/lib2to3/tests/data/py2_test_grammar.py b/Lib/lib2to3/tests/data/py2_test_grammar.py index e5d8a0ff219..dc794e2974c 100644 --- a/Lib/lib2to3/tests/data/py2_test_grammar.py +++ b/Lib/lib2to3/tests/data/py2_test_grammar.py @@ -1,4 +1,4 @@ -# Python 2's Lib/test/test_grammar.py (r54061) +# Python 2's Lib/test/test_grammar.py (r66189) # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. @@ -32,6 +32,8 @@ class TokenTests(unittest.TestCase): self.assertEquals(0xff, 255) self.assertEquals(0377, 255) self.assertEquals(2147483647, 017777777777) + # "0x" is not a valid literal + self.assertRaises(SyntaxError, eval, "0x") from sys import maxint if maxint == 2147483647: self.assertEquals(-2147483647-1, -020000000000) @@ -282,6 +284,18 @@ class GrammarTests(unittest.TestCase): def d32v((x,)): pass d32v((1,)) + # keyword arguments after *arglist + def f(*args, **kwargs): + return args, kwargs + self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), + {'x':2, 'y':5})) + self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") + self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") + + # Check ast errors in *args and *kwargs + check_syntax_error(self, "f(*g(1=2))") + check_syntax_error(self, "f(**g(1=2))") + def testLambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -295,6 +309,7 @@ class GrammarTests(unittest.TestCase): self.assertEquals(l5(1, 2), 5) self.assertEquals(l5(1, 2, 3), 6) check_syntax_error(self, "lambda x: x = 2") + check_syntax_error(self, "lambda (None,): None") ### stmt: simple_stmt | compound_stmt # Tested below @@ -572,6 +587,15 @@ hello world while 0: pass else: pass + # Issue1920: "while 0" is optimized away, + # ensure that the "else" clause is still present. + x = 0 + while 0: + x = 1 + else: + x = 2 + self.assertEquals(x, 2) + def testFor(self): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] for i in 1, 2, 3: pass @@ -602,7 +626,7 @@ hello world def testTry(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr [',' expr]] + ### except_clause: 'except' [expr [('as' | ',') expr]] try: 1/0 except ZeroDivisionError: @@ -611,7 +635,7 @@ hello world pass try: 1/0 except EOFError: pass - except TypeError, msg: pass + except TypeError as msg: pass except RuntimeError, msg: pass except: pass else: pass @@ -770,6 +794,16 @@ hello world def meth1(self): pass def meth2(self, arg): pass def meth3(self, a1, a2): pass + # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + # decorators: decorator+ + # decorated: decorators (classdef | funcdef) + def class_decorator(x): + x.decorated = True + return x + @class_decorator + class G: + pass + self.assertEqual(G.decorated, True) def testListcomps(self): # list comprehension tests diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index bfc77feccfc..9a8846307ec 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -8,7 +8,7 @@ # regression test, the filterwarnings() call has been added to # regrtest.py. -from test.test_support import run_unittest, check_syntax_error +from test.support import run_unittest, check_syntax_error import unittest import sys # testing import * @@ -32,8 +32,10 @@ class TokenTests(unittest.TestCase): self.assertEquals(0o377, 255) self.assertEquals(2147483647, 0o17777777777) self.assertEquals(0b1001, 9) - from sys import maxint - if maxint == 2147483647: + # "0x" is not a valid literal + self.assertRaises(SyntaxError, eval, "0x") + from sys import maxsize + if maxsize == 2147483647: self.assertEquals(-2147483647-1, -0o20000000000) # XXX -2147483648 self.assert_(0o37777777777 > 0) @@ -45,7 +47,7 @@ class TokenTests(unittest.TestCase): x = eval(s) except OverflowError: self.fail("OverflowError on huge integer literal %r" % s) - elif maxint == 9223372036854775807: + elif maxsize == 9223372036854775807: self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000) self.assert_(0o1777777777777777777777 > 0) self.assert_(0xffffffffffffffff > 0) @@ -58,7 +60,7 @@ class TokenTests(unittest.TestCase): except OverflowError: self.fail("OverflowError on huge integer literal %r" % s) else: - self.fail('Weird maxint value %r' % maxint) + self.fail('Weird maxsize value %r' % maxsize) def testLongIntegers(self): x = 0 @@ -263,6 +265,14 @@ class GrammarTests(unittest.TestCase): d22v(*(1, 2, 3, 4)) d22v(1, 2, *(3, 4, 5)) d22v(1, *(2, 3), **{'d': 4}) + + # keyword argument type tests + try: + str('x', **{b'foo':1 }) + except TypeError: + pass + else: + self.fail('Bytes should not work as keyword argument names') # keyword only argument tests def pos0key1(*, key): return key pos0key1(key=100) @@ -274,6 +284,14 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) + # keyword arguments after *arglist + def f(*args, **kwargs): + return args, kwargs + self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), + {'x':2, 'y':5})) + self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") + self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") + # argument annotation tests def f(x) -> list: pass self.assertEquals(f.__annotations__, {'return': list}) @@ -308,6 +326,10 @@ class GrammarTests(unittest.TestCase): def f(*, k=1): return closure def f() -> int: return closure + # Check ast errors in *args and *kwargs + check_syntax_error(self, "f(*g(1=2))") + check_syntax_error(self, "f(**g(1=2))") + def testLambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -321,6 +343,7 @@ class GrammarTests(unittest.TestCase): self.assertEquals(l5(1, 2), 5) self.assertEquals(l5(1, 2, 3), 6) check_syntax_error(self, "lambda x: x = 2") + check_syntax_error(self, "lambda (None,): None") l6 = lambda x, y, *, k=20: x+y+k self.assertEquals(l6(1,2), 1+2+20) self.assertEquals(l6(1,2,k=10), 1+2+10) @@ -438,7 +461,7 @@ class GrammarTests(unittest.TestCase): def testRaise(self): # 'raise' test [',' test] - try: raise RuntimeError, 'just testing' + try: raise RuntimeError('just testing') except RuntimeError: pass try: raise KeyboardInterrupt except KeyboardInterrupt: pass @@ -498,6 +521,15 @@ class GrammarTests(unittest.TestCase): while 0: pass else: pass + # Issue1920: "while 0" is optimized away, + # ensure that the "else" clause is still present. + x = 0 + while 0: + x = 1 + else: + x = 2 + self.assertEquals(x, 2) + def testFor(self): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] for i in 1, 2, 3: pass diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 2dc65d331f7..e62e0f993ec 100755 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -385,6 +385,16 @@ class Test_print(FixerTestCase): a = """print()""" self.check(b, a) + def test_4(self): + # from bug 3000 + b = """print whatever; print""" + a = """print(whatever); print()""" + self.check(b, a) + + def test_5(self): + b = """print; print whatever;""" + a = """print(); print(whatever);""" + def test_tuple(self): b = """print (a, b, c)""" a = """print((a, b, c))""" @@ -2379,6 +2389,15 @@ class Test_numliterals(FixerTestCase): a = """b = 0x12""" self.check(b, a) + def test_comments_and_spacing(self): + b = """b = 0x12L""" + a = """b = 0x12""" + self.check(b, a) + + b = """b = 0755 # spam""" + a = """b = 0o755 # spam""" + self.check(b, a) + def test_unchanged_int(self): s = """5""" self.unchanged(s) @@ -3430,6 +3449,133 @@ class Test_paren(FixerTestCase): s = """[i for i in m]""" self.unchanged(s) +class Test_metaclass(FixerTestCase): + + fixer = 'metaclass' + + def test_unchanged(self): + self.unchanged("class X(): pass") + self.unchanged("class X(object): pass") + self.unchanged("class X(object1, object2): pass") + self.unchanged("class X(object1, object2, object3): pass") + self.unchanged("class X(metaclass=Meta): pass") + self.unchanged("class X(b, arg=23, metclass=Meta): pass") + self.unchanged("class X(b, arg=23, metaclass=Meta, other=42): pass") + + s = """ + class X: + def __metaclass__(self): pass + """ + self.unchanged(s) + + def test_comments(self): + b = """ + class X: + # hi + __metaclass__ = AppleMeta + """ + a = """ + class X(metaclass=AppleMeta): + # hi + pass + """ + self.check(b, a) + + b = """ + class X: + __metaclass__ = Meta + # Bedtime! + """ + a = """ + class X(metaclass=Meta): + pass + # Bedtime! + """ + self.check(b, a) + + def test_meta(self): + # no-parent class, odd body + b = """ + class X(): + __metaclass__ = Q + pass + """ + a = """ + class X(metaclass=Q): + pass + """ + self.check(b, a) + + # one parent class, no body + b = """class X(object): __metaclass__ = Q""" + a = """class X(object, metaclass=Q): pass""" + self.check(b, a) + + + # one parent, simple body + b = """ + class X(object): + __metaclass__ = Meta + bar = 7 + """ + a = """ + class X(object, metaclass=Meta): + bar = 7 + """ + self.check(b, a) + + b = """ + class X: + __metaclass__ = Meta; x = 4; g = 23 + """ + a = """ + class X(metaclass=Meta): + x = 4; g = 23 + """ + self.check(b, a) + + # one parent, simple body, __metaclass__ last + b = """ + class X(object): + bar = 7 + __metaclass__ = Meta + """ + a = """ + class X(object, metaclass=Meta): + bar = 7 + """ + self.check(b, a) + + # redefining __metaclass__ + b = """ + class X(): + __metaclass__ = A + __metaclass__ = B + bar = 7 + """ + a = """ + class X(metaclass=B): + bar = 7 + """ + self.check(b, a) + + # multiple inheritance, simple body + b = """ + class X(clsA, clsB): + __metaclass__ = Meta + bar = 7 + """ + a = """ + class X(clsA, clsB, metaclass=Meta): + bar = 7 + """ + self.check(b, a) + + # keywords in the class statement + b = """class m(a, arg=23): __metaclass__ = Meta""" + a = """class m(a, arg=23, metaclass=Meta): pass""" + self.check(b, a) + if __name__ == "__main__": import __main__