Merged revisions 66191,66418,66438,66445 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r66191 | benjamin.peterson | 2008-09-03 17:00:52 -0500 (Wed, 03 Sep 2008) | 1 line

  update the Grammar file after recent syntax changes
........
  r66418 | benjamin.peterson | 2008-09-12 18:49:48 -0500 (Fri, 12 Sep 2008) | 1 line

  a trival fix to get a few more print corner cases #2899
........
  r66438 | benjamin.peterson | 2008-09-12 21:32:30 -0500 (Fri, 12 Sep 2008) | 5 lines

  add Jack Diederich's fixer for metaclass syntax #2366

  my contribution to this was adding a few tests and fixing a few bugs
  I also reviewed it (Jack is a committer)
........
  r66445 | benjamin.peterson | 2008-09-13 10:50:00 -0500 (Sat, 13 Sep 2008) | 1 line

  add a few more tests concerning int literals and weird spacing
........

Lib/lib2to3/Grammar.txt

@@ -138,7 +138,9 @@ dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
-arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
+arglist: (argument ',')* (argument [',']
+                         |'*' test (',' argument)* [',' '**' test] 
+                         |'**' test)
 argument: test [comp_for] | test '=' test  # Really [keyword '='] test
 
 comp_iter: comp_for | comp_if

Lib/lib2to3/fixes/fix_metaclass.py

@@ -0,0 +1,227 @@
+"""Fixer for __metaclass__ = X -> (metaclass=X) methods.
+
+   The various forms of classef (inherits nothing, inherits once, inherints
+   many) don't parse the same in the CST so we look at ALL classes for
+   a __metaclass__ and if we find one normalize the inherits to all be
+   an arglist.
+
+   For one-liner classes ('class X: pass') there is no indent/dedent so
+   we normalize those into having a suite.
+
+   Moving the __metaclass__ into the classdef can also cause the class
+   body to be empty so there is some special casing for that as well.
+
+   This fixer also tries very hard to keep original indenting and spacing
+   in all those corner cases.
+
+"""
+# Author: Jack Diederich
+
+import os
+
+# Local imports
+from .. import fixer_base
+from ..pygram import token
+from ..fixer_util import Name, syms, Node, Leaf
+
+
+def has_metaclass(parent):
+    """ we have to check the cls_node without changing it.
+        There are two possiblities:
+          1)  clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta')
+          2)  clsdef => simple_stmt => expr_stmt => Leaf('__meta')
+    """
+    for node in parent.children:
+        if node.type == syms.suite:
+            return has_metaclass(node)
+        elif node.type == syms.simple_stmt and node.children:
+            expr_node = node.children[0]
+            if expr_node.type == syms.expr_stmt and expr_node.children:
+                leaf_node = expr_node.children[0]
+                if leaf_node.value == '__metaclass__':
+                    return True
+    return False
+
+
+def fixup_parse_tree(cls_node):
+    """ one-line classes don't get a suite in the parse tree so we add
+        one to normalize the tree
+    """
+    for node in cls_node.children:
+        if node.type == syms.suite:
+            # already in the prefered format, do nothing
+            return
+
+    # !%@#! oneliners have no suite node, we have to fake one up
+    for i, node in enumerate(cls_node.children):
+        if node.type == token.COLON:
+            break
+    else:
+        raise ValueError("No class suite and no ':'!")
+
+    # move everything into a suite node
+    suite = Node(syms.suite, [])
+    while cls_node.children[i+1:]:
+        move_node = cls_node.children[i+1]
+        suite.append_child(move_node.clone())
+        move_node.remove()
+    cls_node.append_child(suite)
+    node = suite
+
+
+def fixup_simple_stmt(parent, i, stmt_node):
+    """ if there is a semi-colon all the parts count as part of the same
+        simple_stmt.  We just want the __metaclass__ part so we move
+        everything efter the semi-colon into its own simple_stmt node
+    """
+    for semi_ind, node in enumerate(stmt_node.children):
+        if node.type == token.SEMI: # *sigh*
+            break
+    else:
+        return
+
+    node.remove() # kill the semicolon
+    new_expr = Node(syms.expr_stmt, [])
+    new_stmt = Node(syms.simple_stmt, [new_expr])
+    while stmt_node.children[semi_ind:]:
+        move_node = stmt_node.children[semi_ind]
+        new_expr.append_child(move_node.clone())
+        move_node.remove()
+    parent.insert_child(i, new_stmt)
+    new_leaf1 = new_stmt.children[0].children[0]
+    old_leaf1 = stmt_node.children[0].children[0]
+    new_leaf1.set_prefix(old_leaf1.get_prefix())
+
+
+def remove_trailing_newline(node):
+    if node.children and node.children[-1].type == token.NEWLINE:
+        node.children[-1].remove()
+
+
+def find_metas(cls_node):
+    # find the suite node (Mmm, sweet nodes)
+    for node in cls_node.children:
+        if node.type == syms.suite:
+            break
+    else:
+        raise ValueError("No class suite!")
+
+    # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ]
+    for i, simple_node in list(enumerate(node.children)):
+        if simple_node.type == syms.simple_stmt and simple_node.children:
+            expr_node = simple_node.children[0]
+            if expr_node.type == syms.expr_stmt and expr_node.children:
+                leaf_node = expr_node.children[0]
+                if leaf_node.value == '__metaclass__':
+                    fixup_simple_stmt(node, i, simple_node)
+                    remove_trailing_newline(simple_node)
+                    yield (node, i, simple_node)
+
+
+def fixup_indent(suite):
+    """ If an INDENT is followed by a thing with a prefix then nuke the prefix
+        Otherwise we get in trouble when removing __metaclass__ at suite start
+    """
+    kids = suite.children[::-1]
+    # find the first indent
+    while kids:
+        node = kids.pop()
+        if node.type == token.INDENT:
+            break
+
+    # find the first Leaf
+    while kids:
+        node = kids.pop()
+        if isinstance(node, Leaf) and node.type != token.DEDENT:
+            if node.prefix:
+                node.set_prefix('')
+            return
+        else:
+            kids.extend(node.children[::-1])
+
+
+class FixMetaclass(fixer_base.BaseFix):
+
+    PATTERN = """
+    classdef<any*>
+    """
+
+    def transform(self, node, results):
+        if not has_metaclass(node):
+            return node
+
+        fixup_parse_tree(node)
+
+        # find metaclasses, keep the last one
+        last_metaclass = None
+        for suite, i, stmt in find_metas(node):
+            last_metaclass = stmt
+            stmt.remove()
+
+        text_type = node.children[0].type # always Leaf(nnn, 'class')
+
+        # figure out what kind of classdef we have
+        if len(node.children) == 7:
+            # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite])
+            #                 0        1       2    3        4    5    6
+            if node.children[3].type == syms.arglist:
+                arglist = node.children[3]
+            # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite])
+            elif isinstance(node.children[3], Leaf):
+                parent = node.children[3].clone()
+                arglist = Node(syms.arglist, [parent])
+                node.set_child(3, arglist)
+            else:
+                raise ValueError("Unexpected class inheritance arglist")
+        elif len(node.children) == 6:
+            # Node(classdef, ['class', 'name', '(',  ')', ':', suite])
+            #                 0        1       2     3    4    5
+            arglist = Node(syms.arglist, [])
+            node.insert_child(3, arglist)
+        elif len(node.children) == 4:
+            # Node(classdef, ['class', 'name', ':', suite])
+            #                 0        1       2    3
+            arglist = Node(syms.arglist, [])
+            node.insert_child(2, Leaf(token.RPAR, ')'))
+            node.insert_child(2, arglist)
+            node.insert_child(2, Leaf(token.LPAR, '('))
+        else:
+            raise ValueError("Unexpected class definition")
+
+        # now stick the metaclass in the arglist
+        meta_txt = last_metaclass.children[0].children[0]
+        meta_txt.value = 'metaclass'
+        orig_meta_prefix = meta_txt.get_prefix()
+
+        if arglist.children:
+            arglist.append_child(Leaf(token.COMMA, ','))
+            meta_txt.set_prefix(' ')
+        else:
+            meta_txt.set_prefix('')
+
+        # compact the expression "metaclass = Meta" -> "metaclass=Meta"
+        expr_stmt = last_metaclass.children[0]
+        assert expr_stmt.type == syms.expr_stmt
+        expr_stmt.children[1].set_prefix('')
+        expr_stmt.children[2].set_prefix('')
+
+        arglist.append_child(last_metaclass)
+
+        fixup_indent(suite)
+
+        # check for empty suite
+        if not suite.children:
+            # one-liner that was just __metaclass_
+            suite.remove()
+            pass_leaf = Leaf(text_type, 'pass')
+            pass_leaf.set_prefix(orig_meta_prefix)
+            node.append_child(pass_leaf)
+            node.append_child(Leaf(token.NEWLINE, os.linesep))
+
+        elif len(suite.children) > 1 and \
+                 (suite.children[-2].type == token.INDENT and
+                  suite.children[-1].type == token.DEDENT):
+            # there was only one line in the class body and it was __metaclass__
+            pass_leaf = Leaf(text_type, 'pass')
+            suite.insert_child(-1, pass_leaf)
+            suite.insert_child(-1, Leaf(token.NEWLINE, os.linesep))

Lib/lib2to3/fixes/fix_print.py

@@ -29,7 +29,7 @@ parend_expr = patcomp.compile_pattern(
 class FixPrint(fixer_base.ConditionalFix):
 
     PATTERN = """
-              simple_stmt< bare='print' any > | print_stmt
+              simple_stmt< any* bare='print' any* > | print_stmt
               """
 
     skip_on = '__future__.print_function'

Lib/lib2to3/tests/data/py2_test_grammar.py

@@ -1,4 +1,4 @@
-# Python 2's Lib/test/test_grammar.py (r54061)
+# Python 2's Lib/test/test_grammar.py (r66189)
 
 # Python test set -- part 1, grammar.
 # This just tests whether the parser accepts them all.
@@ -32,6 +32,8 @@ class TokenTests(unittest.TestCase):
         self.assertEquals(0xff, 255)
         self.assertEquals(0377, 255)
         self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
         from sys import maxint
         if maxint == 2147483647:
             self.assertEquals(-2147483647-1, -020000000000)
@@ -282,6 +284,18 @@ class GrammarTests(unittest.TestCase):
         def d32v((x,)): pass
         d32v((1,))
 
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
     def testLambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0
@@ -295,6 +309,7 @@ class GrammarTests(unittest.TestCase):
         self.assertEquals(l5(1, 2), 5)
         self.assertEquals(l5(1, 2, 3), 6)
         check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
 
     ### stmt: simple_stmt | compound_stmt
     # Tested below
@@ -572,6 +587,15 @@ hello world
         while 0: pass
         else: pass
 
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
     def testFor(self):
         # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
         for i in 1, 2, 3: pass
@@ -602,7 +626,7 @@ hello world
     def testTry(self):
         ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
         ###         | 'try' ':' suite 'finally' ':' suite
-        ### except_clause: 'except' [expr [',' expr]]
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
         try:
             1/0
         except ZeroDivisionError:
@@ -611,7 +635,7 @@ hello world
             pass
         try: 1/0
         except EOFError: pass
-        except TypeError, msg: pass
+        except TypeError as msg: pass
         except RuntimeError, msg: pass
         except: pass
         else: pass
@@ -770,6 +794,16 @@ hello world
             def meth1(self): pass
             def meth2(self, arg): pass
             def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
 
     def testListcomps(self):
         # list comprehension tests

Lib/lib2to3/tests/data/py3_test_grammar.py

@@ -8,7 +8,7 @@
 # regression test, the filterwarnings() call has been added to
 # regrtest.py.
 
-from test.test_support import run_unittest, check_syntax_error
+from test.support import run_unittest, check_syntax_error
 import unittest
 import sys
 # testing import *
@@ -32,8 +32,10 @@ class TokenTests(unittest.TestCase):
         self.assertEquals(0o377, 255)
         self.assertEquals(2147483647, 0o17777777777)
         self.assertEquals(0b1001, 9)
-        from sys import maxint
+        # "0x" is not a valid literal
-        if maxint == 2147483647:
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
             self.assertEquals(-2147483647-1, -0o20000000000)
             # XXX -2147483648
             self.assert_(0o37777777777 > 0)
@@ -45,7 +47,7 @@ class TokenTests(unittest.TestCase):
                     x = eval(s)
                 except OverflowError:
                     self.fail("OverflowError on huge integer literal %r" % s)
-        elif maxint == 9223372036854775807:
+        elif maxsize == 9223372036854775807:
             self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
             self.assert_(0o1777777777777777777777 > 0)
             self.assert_(0xffffffffffffffff > 0)
@@ -58,7 +60,7 @@ class TokenTests(unittest.TestCase):
                 except OverflowError:
                     self.fail("OverflowError on huge integer literal %r" % s)
         else:
-            self.fail('Weird maxint value %r' % maxint)
+            self.fail('Weird maxsize value %r' % maxsize)
 
     def testLongIntegers(self):
         x = 0
@@ -263,6 +265,14 @@ class GrammarTests(unittest.TestCase):
         d22v(*(1, 2, 3, 4))
         d22v(1, 2, *(3, 4, 5))
         d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
         # keyword only argument tests
         def pos0key1(*, key): return key
         pos0key1(key=100)
@@ -274,6 +284,14 @@ class GrammarTests(unittest.TestCase):
         pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
         pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
 
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
         # argument annotation tests
         def f(x) -> list: pass
         self.assertEquals(f.__annotations__, {'return': list})
@@ -308,6 +326,10 @@ class GrammarTests(unittest.TestCase):
         def f(*, k=1): return closure
         def f() -> int: return closure
 
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
     def testLambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0
@@ -321,6 +343,7 @@ class GrammarTests(unittest.TestCase):
         self.assertEquals(l5(1, 2), 5)
         self.assertEquals(l5(1, 2, 3), 6)
         check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
         l6 = lambda x, y, *, k=20: x+y+k
         self.assertEquals(l6(1,2), 1+2+20)
         self.assertEquals(l6(1,2,k=10), 1+2+10)
@@ -438,7 +461,7 @@ class GrammarTests(unittest.TestCase):
 
     def testRaise(self):
         # 'raise' test [',' test]
-        try: raise RuntimeError, 'just testing'
+        try: raise RuntimeError('just testing')
         except RuntimeError: pass
         try: raise KeyboardInterrupt
         except KeyboardInterrupt: pass
@@ -498,6 +521,15 @@ class GrammarTests(unittest.TestCase):
         while 0: pass
         else: pass
 
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
     def testFor(self):
         # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
         for i in 1, 2, 3: pass

Lib/lib2to3/tests/test_fixers.py

@@ -385,6 +385,16 @@ class Test_print(FixerTestCase):
         a = """print()"""
         self.check(b, a)
 
+    def test_4(self):
+        # from bug 3000
+        b = """print whatever; print"""
+        a = """print(whatever); print()"""
+        self.check(b, a)
+
+    def test_5(self):
+        b = """print; print whatever;"""
+        a = """print(); print(whatever);"""
+
     def test_tuple(self):
         b = """print (a, b, c)"""
         a = """print((a, b, c))"""
@@ -2379,6 +2389,15 @@ class Test_numliterals(FixerTestCase):
         a = """b = 0x12"""
         self.check(b, a)
 
+    def test_comments_and_spacing(self):
+        b = """b =   0x12L"""
+        a = """b =   0x12"""
+        self.check(b, a)
+
+        b = """b = 0755 # spam"""
+        a = """b = 0o755 # spam"""
+        self.check(b, a)
+
     def test_unchanged_int(self):
         s = """5"""
         self.unchanged(s)
@@ -3430,6 +3449,133 @@ class Test_paren(FixerTestCase):
         s = """[i for i in m]"""
         self.unchanged(s)
 
+class Test_metaclass(FixerTestCase):
+
+    fixer = 'metaclass'
+
+    def test_unchanged(self):
+        self.unchanged("class X(): pass")
+        self.unchanged("class X(object): pass")
+        self.unchanged("class X(object1, object2): pass")
+        self.unchanged("class X(object1, object2, object3): pass")
+        self.unchanged("class X(metaclass=Meta): pass")
+        self.unchanged("class X(b, arg=23, metclass=Meta): pass")
+        self.unchanged("class X(b, arg=23, metaclass=Meta, other=42): pass")
+
+        s = """
+        class X:
+            def __metaclass__(self): pass
+        """
+        self.unchanged(s)
+
+    def test_comments(self):
+        b = """
+        class X:
+            # hi
+            __metaclass__ = AppleMeta
+        """
+        a = """
+        class X(metaclass=AppleMeta):
+            # hi
+            pass
+        """
+        self.check(b, a)
+
+        b = """
+        class X:
+            __metaclass__ = Meta
+            # Bedtime!
+        """
+        a = """
+        class X(metaclass=Meta):
+            pass
+            # Bedtime!
+        """
+        self.check(b, a)
+
+    def test_meta(self):
+        # no-parent class, odd body
+        b = """
+        class X():
+            __metaclass__ = Q
+            pass
+        """
+        a = """
+        class X(metaclass=Q):
+            pass
+        """
+        self.check(b, a)
+
+        # one parent class, no body
+        b = """class X(object): __metaclass__ = Q"""
+        a = """class X(object, metaclass=Q): pass"""
+        self.check(b, a)
+
+
+        # one parent, simple body
+        b = """
+        class X(object):
+            __metaclass__ = Meta
+            bar = 7
+        """
+        a = """
+        class X(object, metaclass=Meta):
+            bar = 7
+        """
+        self.check(b, a)
+
+        b = """
+        class X:
+            __metaclass__ = Meta; x = 4; g = 23
+        """
+        a = """
+        class X(metaclass=Meta):
+            x = 4; g = 23
+        """
+        self.check(b, a)
+
+        # one parent, simple body, __metaclass__ last
+        b = """
+        class X(object):
+            bar = 7
+            __metaclass__ = Meta
+        """
+        a = """
+        class X(object, metaclass=Meta):
+            bar = 7
+        """
+        self.check(b, a)
+
+        # redefining __metaclass__
+        b = """
+        class X():
+            __metaclass__ = A
+            __metaclass__ = B
+            bar = 7
+        """
+        a = """
+        class X(metaclass=B):
+            bar = 7
+        """
+        self.check(b, a)
+
+        # multiple inheritance, simple body
+        b = """
+        class X(clsA, clsB):
+            __metaclass__ = Meta
+            bar = 7
+        """
+        a = """
+        class X(clsA, clsB, metaclass=Meta):
+            bar = 7
+        """
+        self.check(b, a)
+
+        # keywords in the class statement
+        b = """class m(a, arg=23): __metaclass__ = Meta"""
+        a = """class m(a, arg=23, metaclass=Meta): pass"""
+        self.check(b, a)
+
 
 if __name__ == "__main__":
     import __main__