Merged revisions 66191,66418,66438,66445 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r66191 | benjamin.peterson | 2008-09-03 17:00:52 -0500 (Wed, 03 Sep 2008) | 1 line

  update the Grammar file after recent syntax changes
........
  r66418 | benjamin.peterson | 2008-09-12 18:49:48 -0500 (Fri, 12 Sep 2008) | 1 line

  a trival fix to get a few more print corner cases #2899
........
  r66438 | benjamin.peterson | 2008-09-12 21:32:30 -0500 (Fri, 12 Sep 2008) | 5 lines

  add Jack Diederich's fixer for metaclass syntax #2366

  my contribution to this was adding a few tests and fixing a few bugs
  I also reviewed it (Jack is a committer)
........
  r66445 | benjamin.peterson | 2008-09-13 10:50:00 -0500 (Sat, 13 Sep 2008) | 1 line

  add a few more tests concerning int literals and weird spacing
........
This commit is contained in:
Benjamin Peterson 2008-09-13 17:43:19 +00:00
parent 9af9498c6e
commit d51e07f989
6 changed files with 452 additions and 11 deletions

View File

@ -138,7 +138,9 @@ dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
argument: test [comp_for] | test '=' test # Really [keyword '='] test
comp_iter: comp_for | comp_if

View File

@ -0,0 +1,227 @@
"""Fixer for __metaclass__ = X -> (metaclass=X) methods.
The various forms of classef (inherits nothing, inherits once, inherints
many) don't parse the same in the CST so we look at ALL classes for
a __metaclass__ and if we find one normalize the inherits to all be
an arglist.
For one-liner classes ('class X: pass') there is no indent/dedent so
we normalize those into having a suite.
Moving the __metaclass__ into the classdef can also cause the class
body to be empty so there is some special casing for that as well.
This fixer also tries very hard to keep original indenting and spacing
in all those corner cases.
"""
# Author: Jack Diederich
import os
# Local imports
from .. import fixer_base
from ..pygram import token
from ..fixer_util import Name, syms, Node, Leaf
def has_metaclass(parent):
""" we have to check the cls_node without changing it.
There are two possiblities:
1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta')
2) clsdef => simple_stmt => expr_stmt => Leaf('__meta')
"""
for node in parent.children:
if node.type == syms.suite:
return has_metaclass(node)
elif node.type == syms.simple_stmt and node.children:
expr_node = node.children[0]
if expr_node.type == syms.expr_stmt and expr_node.children:
leaf_node = expr_node.children[0]
if leaf_node.value == '__metaclass__':
return True
return False
def fixup_parse_tree(cls_node):
""" one-line classes don't get a suite in the parse tree so we add
one to normalize the tree
"""
for node in cls_node.children:
if node.type == syms.suite:
# already in the prefered format, do nothing
return
# !%@#! oneliners have no suite node, we have to fake one up
for i, node in enumerate(cls_node.children):
if node.type == token.COLON:
break
else:
raise ValueError("No class suite and no ':'!")
# move everything into a suite node
suite = Node(syms.suite, [])
while cls_node.children[i+1:]:
move_node = cls_node.children[i+1]
suite.append_child(move_node.clone())
move_node.remove()
cls_node.append_child(suite)
node = suite
def fixup_simple_stmt(parent, i, stmt_node):
""" if there is a semi-colon all the parts count as part of the same
simple_stmt. We just want the __metaclass__ part so we move
everything efter the semi-colon into its own simple_stmt node
"""
for semi_ind, node in enumerate(stmt_node.children):
if node.type == token.SEMI: # *sigh*
break
else:
return
node.remove() # kill the semicolon
new_expr = Node(syms.expr_stmt, [])
new_stmt = Node(syms.simple_stmt, [new_expr])
while stmt_node.children[semi_ind:]:
move_node = stmt_node.children[semi_ind]
new_expr.append_child(move_node.clone())
move_node.remove()
parent.insert_child(i, new_stmt)
new_leaf1 = new_stmt.children[0].children[0]
old_leaf1 = stmt_node.children[0].children[0]
new_leaf1.set_prefix(old_leaf1.get_prefix())
def remove_trailing_newline(node):
if node.children and node.children[-1].type == token.NEWLINE:
node.children[-1].remove()
def find_metas(cls_node):
# find the suite node (Mmm, sweet nodes)
for node in cls_node.children:
if node.type == syms.suite:
break
else:
raise ValueError("No class suite!")
# look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ]
for i, simple_node in list(enumerate(node.children)):
if simple_node.type == syms.simple_stmt and simple_node.children:
expr_node = simple_node.children[0]
if expr_node.type == syms.expr_stmt and expr_node.children:
leaf_node = expr_node.children[0]
if leaf_node.value == '__metaclass__':
fixup_simple_stmt(node, i, simple_node)
remove_trailing_newline(simple_node)
yield (node, i, simple_node)
def fixup_indent(suite):
""" If an INDENT is followed by a thing with a prefix then nuke the prefix
Otherwise we get in trouble when removing __metaclass__ at suite start
"""
kids = suite.children[::-1]
# find the first indent
while kids:
node = kids.pop()
if node.type == token.INDENT:
break
# find the first Leaf
while kids:
node = kids.pop()
if isinstance(node, Leaf) and node.type != token.DEDENT:
if node.prefix:
node.set_prefix('')
return
else:
kids.extend(node.children[::-1])
class FixMetaclass(fixer_base.BaseFix):
PATTERN = """
classdef<any*>
"""
def transform(self, node, results):
if not has_metaclass(node):
return node
fixup_parse_tree(node)
# find metaclasses, keep the last one
last_metaclass = None
for suite, i, stmt in find_metas(node):
last_metaclass = stmt
stmt.remove()
text_type = node.children[0].type # always Leaf(nnn, 'class')
# figure out what kind of classdef we have
if len(node.children) == 7:
# Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite])
# 0 1 2 3 4 5 6
if node.children[3].type == syms.arglist:
arglist = node.children[3]
# Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite])
elif isinstance(node.children[3], Leaf):
parent = node.children[3].clone()
arglist = Node(syms.arglist, [parent])
node.set_child(3, arglist)
else:
raise ValueError("Unexpected class inheritance arglist")
elif len(node.children) == 6:
# Node(classdef, ['class', 'name', '(', ')', ':', suite])
# 0 1 2 3 4 5
arglist = Node(syms.arglist, [])
node.insert_child(3, arglist)
elif len(node.children) == 4:
# Node(classdef, ['class', 'name', ':', suite])
# 0 1 2 3
arglist = Node(syms.arglist, [])
node.insert_child(2, Leaf(token.RPAR, ')'))
node.insert_child(2, arglist)
node.insert_child(2, Leaf(token.LPAR, '('))
else:
raise ValueError("Unexpected class definition")
# now stick the metaclass in the arglist
meta_txt = last_metaclass.children[0].children[0]
meta_txt.value = 'metaclass'
orig_meta_prefix = meta_txt.get_prefix()
if arglist.children:
arglist.append_child(Leaf(token.COMMA, ','))
meta_txt.set_prefix(' ')
else:
meta_txt.set_prefix('')
# compact the expression "metaclass = Meta" -> "metaclass=Meta"
expr_stmt = last_metaclass.children[0]
assert expr_stmt.type == syms.expr_stmt
expr_stmt.children[1].set_prefix('')
expr_stmt.children[2].set_prefix('')
arglist.append_child(last_metaclass)
fixup_indent(suite)
# check for empty suite
if not suite.children:
# one-liner that was just __metaclass_
suite.remove()
pass_leaf = Leaf(text_type, 'pass')
pass_leaf.set_prefix(orig_meta_prefix)
node.append_child(pass_leaf)
node.append_child(Leaf(token.NEWLINE, os.linesep))
elif len(suite.children) > 1 and \
(suite.children[-2].type == token.INDENT and
suite.children[-1].type == token.DEDENT):
# there was only one line in the class body and it was __metaclass__
pass_leaf = Leaf(text_type, 'pass')
suite.insert_child(-1, pass_leaf)
suite.insert_child(-1, Leaf(token.NEWLINE, os.linesep))

View File

@ -29,7 +29,7 @@ parend_expr = patcomp.compile_pattern(
class FixPrint(fixer_base.ConditionalFix):
PATTERN = """
simple_stmt< bare='print' any > | print_stmt
simple_stmt< any* bare='print' any* > | print_stmt
"""
skip_on = '__future__.print_function'

View File

@ -1,4 +1,4 @@
# Python 2's Lib/test/test_grammar.py (r54061)
# Python 2's Lib/test/test_grammar.py (r66189)
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
@ -32,6 +32,8 @@ class TokenTests(unittest.TestCase):
self.assertEquals(0xff, 255)
self.assertEquals(0377, 255)
self.assertEquals(2147483647, 017777777777)
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxint
if maxint == 2147483647:
self.assertEquals(-2147483647-1, -020000000000)
@ -282,6 +284,18 @@ class GrammarTests(unittest.TestCase):
def d32v((x,)): pass
d32v((1,))
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
@ -295,6 +309,7 @@ class GrammarTests(unittest.TestCase):
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
### stmt: simple_stmt | compound_stmt
# Tested below
@ -572,6 +587,15 @@ hello world
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass
@ -602,7 +626,7 @@ hello world
def testTry(self):
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
### | 'try' ':' suite 'finally' ':' suite
### except_clause: 'except' [expr [',' expr]]
### except_clause: 'except' [expr [('as' | ',') expr]]
try:
1/0
except ZeroDivisionError:
@ -611,7 +635,7 @@ hello world
pass
try: 1/0
except EOFError: pass
except TypeError, msg: pass
except TypeError as msg: pass
except RuntimeError, msg: pass
except: pass
else: pass
@ -770,6 +794,16 @@ hello world
def meth1(self): pass
def meth2(self, arg): pass
def meth3(self, a1, a2): pass
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
# decorators: decorator+
# decorated: decorators (classdef | funcdef)
def class_decorator(x):
x.decorated = True
return x
@class_decorator
class G:
pass
self.assertEqual(G.decorated, True)
def testListcomps(self):
# list comprehension tests

View File

@ -8,7 +8,7 @@
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.test_support import run_unittest, check_syntax_error
from test.support import run_unittest, check_syntax_error
import unittest
import sys
# testing import *
@ -32,8 +32,10 @@ class TokenTests(unittest.TestCase):
self.assertEquals(0o377, 255)
self.assertEquals(2147483647, 0o17777777777)
self.assertEquals(0b1001, 9)
from sys import maxint
if maxint == 2147483647:
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxsize
if maxsize == 2147483647:
self.assertEquals(-2147483647-1, -0o20000000000)
# XXX -2147483648
self.assert_(0o37777777777 > 0)
@ -45,7 +47,7 @@ class TokenTests(unittest.TestCase):
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
elif maxint == 9223372036854775807:
elif maxsize == 9223372036854775807:
self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
self.assert_(0o1777777777777777777777 > 0)
self.assert_(0xffffffffffffffff > 0)
@ -58,7 +60,7 @@ class TokenTests(unittest.TestCase):
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
else:
self.fail('Weird maxint value %r' % maxint)
self.fail('Weird maxsize value %r' % maxsize)
def testLongIntegers(self):
x = 0
@ -263,6 +265,14 @@ class GrammarTests(unittest.TestCase):
d22v(*(1, 2, 3, 4))
d22v(1, 2, *(3, 4, 5))
d22v(1, *(2, 3), **{'d': 4})
# keyword argument type tests
try:
str('x', **{b'foo':1 })
except TypeError:
pass
else:
self.fail('Bytes should not work as keyword argument names')
# keyword only argument tests
def pos0key1(*, key): return key
pos0key1(key=100)
@ -274,6 +284,14 @@ class GrammarTests(unittest.TestCase):
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# argument annotation tests
def f(x) -> list: pass
self.assertEquals(f.__annotations__, {'return': list})
@ -308,6 +326,10 @@ class GrammarTests(unittest.TestCase):
def f(*, k=1): return closure
def f() -> int: return closure
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
@ -321,6 +343,7 @@ class GrammarTests(unittest.TestCase):
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
l6 = lambda x, y, *, k=20: x+y+k
self.assertEquals(l6(1,2), 1+2+20)
self.assertEquals(l6(1,2,k=10), 1+2+10)
@ -438,7 +461,7 @@ class GrammarTests(unittest.TestCase):
def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError, 'just testing'
try: raise RuntimeError('just testing')
except RuntimeError: pass
try: raise KeyboardInterrupt
except KeyboardInterrupt: pass
@ -498,6 +521,15 @@ class GrammarTests(unittest.TestCase):
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass

View File

@ -385,6 +385,16 @@ class Test_print(FixerTestCase):
a = """print()"""
self.check(b, a)
def test_4(self):
# from bug 3000
b = """print whatever; print"""
a = """print(whatever); print()"""
self.check(b, a)
def test_5(self):
b = """print; print whatever;"""
a = """print(); print(whatever);"""
def test_tuple(self):
b = """print (a, b, c)"""
a = """print((a, b, c))"""
@ -2379,6 +2389,15 @@ class Test_numliterals(FixerTestCase):
a = """b = 0x12"""
self.check(b, a)
def test_comments_and_spacing(self):
b = """b = 0x12L"""
a = """b = 0x12"""
self.check(b, a)
b = """b = 0755 # spam"""
a = """b = 0o755 # spam"""
self.check(b, a)
def test_unchanged_int(self):
s = """5"""
self.unchanged(s)
@ -3430,6 +3449,133 @@ class Test_paren(FixerTestCase):
s = """[i for i in m]"""
self.unchanged(s)
class Test_metaclass(FixerTestCase):
fixer = 'metaclass'
def test_unchanged(self):
self.unchanged("class X(): pass")
self.unchanged("class X(object): pass")
self.unchanged("class X(object1, object2): pass")
self.unchanged("class X(object1, object2, object3): pass")
self.unchanged("class X(metaclass=Meta): pass")
self.unchanged("class X(b, arg=23, metclass=Meta): pass")
self.unchanged("class X(b, arg=23, metaclass=Meta, other=42): pass")
s = """
class X:
def __metaclass__(self): pass
"""
self.unchanged(s)
def test_comments(self):
b = """
class X:
# hi
__metaclass__ = AppleMeta
"""
a = """
class X(metaclass=AppleMeta):
# hi
pass
"""
self.check(b, a)
b = """
class X:
__metaclass__ = Meta
# Bedtime!
"""
a = """
class X(metaclass=Meta):
pass
# Bedtime!
"""
self.check(b, a)
def test_meta(self):
# no-parent class, odd body
b = """
class X():
__metaclass__ = Q
pass
"""
a = """
class X(metaclass=Q):
pass
"""
self.check(b, a)
# one parent class, no body
b = """class X(object): __metaclass__ = Q"""
a = """class X(object, metaclass=Q): pass"""
self.check(b, a)
# one parent, simple body
b = """
class X(object):
__metaclass__ = Meta
bar = 7
"""
a = """
class X(object, metaclass=Meta):
bar = 7
"""
self.check(b, a)
b = """
class X:
__metaclass__ = Meta; x = 4; g = 23
"""
a = """
class X(metaclass=Meta):
x = 4; g = 23
"""
self.check(b, a)
# one parent, simple body, __metaclass__ last
b = """
class X(object):
bar = 7
__metaclass__ = Meta
"""
a = """
class X(object, metaclass=Meta):
bar = 7
"""
self.check(b, a)
# redefining __metaclass__
b = """
class X():
__metaclass__ = A
__metaclass__ = B
bar = 7
"""
a = """
class X(metaclass=B):
bar = 7
"""
self.check(b, a)
# multiple inheritance, simple body
b = """
class X(clsA, clsB):
__metaclass__ = Meta
bar = 7
"""
a = """
class X(clsA, clsB, metaclass=Meta):
bar = 7
"""
self.check(b, a)
# keywords in the class statement
b = """class m(a, arg=23): __metaclass__ = Meta"""
a = """class m(a, arg=23, metaclass=Meta): pass"""
self.check(b, a)
if __name__ == "__main__":
import __main__