traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merged revisions 61598-61599,61601 via svnmerge from 

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r61598 | david.wolever | 2008-03-18 23:58:33 -0500 (Di, 18 Mär 2008) | 1 line

  Added fixer for zip, and refactored a bit of code in the process.  Closing #2171.
........
  r61599 | david.wolever | 2008-03-19 00:04:26 -0500 (Mi, 19 Mär 2008) | 3 lines

  Removed a bunch of duplicate code -- it's in util now.
........
  r61601 | martin.v.loewis | 2008-03-19 00:21:12 -0500 (Mi, 19 Mär 2008) | 2 lines

  Fix whitespace.
........
This commit is contained in:
Martin v. Löwis 2008-03-19 05:22:42 +00:00
parent aad2b61f44
commit ab41b370a3
24 changed files with 249 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Lib/lib2to3/fixes/basefix.py
View File

@ -87,7 +87,7 @@ class BaseFix(object):
Args:
node: the root of the parse tree that matched the fixer.
results: a dict mapping symbolic names to part of the match.
Returns:
None, or a node that is a modified copy of the
argument node. The node argument may also be modified in-place to
@ -146,7 +146,7 @@ class BaseFix(object):
def start_tree(self, tree, filename):
"""Some fixers need to maintain tree-wide state.
This method is called once, at the start of tree fix-up.
tree - the root node of the tree to be processed.
filename - the name of the file the tree came from.
"""
@ -158,7 +158,7 @@ class BaseFix(object):
def finish_tree(self, tree, filename):
"""Some fixers need to maintain tree-wide state.
This method is called once, at the conclusion of tree fix-up.
tree - the root node of the tree to be processed.
filename - the name of the file the tree came from.
"""

14
Lib/lib2to3/fixes/fix_except.py
View File

@ -3,19 +3,19 @@
The following cases will be converted:
- "except E, T:" where T is a name:
except E as T:
- "except E, T:" where T is not a name, tuple or list:
except E as t:
T = t
This is done because the target of an "except" clause must be a
name.
- "except E, T:" where T is a tuple or list literal:
except E as t:
T = t.args
"""
@ -39,7 +39,7 @@ class FixExcept(basefix.BaseFix):
try_stmt< 'try' ':' suite
cleanup=((except_clause ':' suite)+ ['else' ':' suite]
['finally' ':' suite]
| 'finally' ':' suite) >
| 'finally' ':' suite) >
"""
def transform(self, node, results):

36
Lib/lib2to3/fixes/fix_filter.py
View File

@ -14,11 +14,9 @@ Python 2.6 figure it out.
"""
# Local imports
from .. import pytree
from .. import patcomp
from ..pgen2 import token
from . import basefix
from .util import Name, Call, ListComp, attr_chain, does_tree_import
from .util import Name, Call, ListComp, does_tree_import, in_special_context
class FixFilter(basefix.BaseFix):
@ -85,35 +83,3 @@ class FixFilter(basefix.BaseFix):
new = Call(Name("list"), [new])
new.set_prefix(node.get_prefix())
return new
P0 = """for_stmt< 'for' any 'in' node=any ':' any* >
| comp_for< 'for' any 'in' node=any any* >
"""
p0 = patcomp.compile_pattern(P0)
P1 = """
power<
( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
'any' | 'all' | (any* trailer< '.' 'join' >) )
trailer< '(' node=any ')' >
any*
>
"""
p1 = patcomp.compile_pattern(P1)
P2 = """
power<
'sorted'
trailer< '(' arglist<node=any any*> ')' >
any*
>
"""
p2 = patcomp.compile_pattern(P2)
def in_special_context(node):
patterns = [p0, p1, p2]
for pattern, parent in zip(patterns, attr_chain(node, "parent")):
results = {}
if pattern.match(parent, results) and results["node"] is node:
return True
return False

5
Lib/lib2to3/fixes/fix_future.py
View File

@ -2,15 +2,14 @@
from __future__ import foo is replaced with an empty line.
"""
# Author: Christian Heimes
# Author: Christian Heimes
# Local imports
from . import basefix
from .util import BlankLine
from .util import BlankLine
class FixFuture(basefix.BaseFix):
PATTERN = """import_from< 'from' module_name="__future__" 'import' any >"""
def transform(self, node, results):
return BlankLine()

6
Lib/lib2to3/fixes/fix_has_key.py
View File

@ -18,14 +18,14 @@ CAVEATS:
m = d.has_key
if m(k):
...
Only *calls* to has_key() are converted. While it is possible to
convert the above to something like
m = d.__contains__
if m(k):
...
this is currently not done.
"""

4
Lib/lib2to3/fixes/fix_idioms.py
View File

@ -5,7 +5,7 @@
type(x) is T -> isinstance(x, T)
type(x) != T -> not isinstance(x, T)
type(x) is not T -> not isinstance(x, T)
* Change "while 1:" into "while True:".
* Change both
@ -19,7 +19,7 @@ and the more general
v = EXPR
v.sort()
foo(v)
into
v = sorted(EXPR)

4
Lib/lib2to3/fixes/fix_imports.py
View File

@ -16,7 +16,7 @@ builtin_names = [name for name in dir(__builtin__)
MAPPING = {"StringIO": ("io", ["StringIO"]),
"cStringIO": ("io", ["StringIO"]),
"__builtin__" : ("builtins", builtin_names),
"__builtin__" : ("builtins", builtin_names),
}
@ -86,4 +86,4 @@ class FixImports(basefix.BaseFix):
bare_name = bare_name[0]
new_name = self.replace.get(bare_name.value)
if new_name:
bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))
bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))

38
Lib/lib2to3/fixes/fix_map.py
View File

@ -20,11 +20,9 @@ soon as the shortest argument is exhausted.
"""
# Local imports
from .. import pytree
from .. import patcomp
from ..pgen2 import token
from . import basefix
from .util import Name, Call, ListComp, attr_chain, does_tree_import
from .util import Name, Call, ListComp, does_tree_import, in_special_context
from ..pygram import python_symbols as syms
class FixMap(basefix.BaseFix):
@ -71,7 +69,7 @@ class FixMap(basefix.BaseFix):
# If a future map has been imported for this file, we won't
# be making any modifications
return
if node.parent.type == syms.simple_stmt:
self.warning(node, "You should use a for loop here")
new = node.clone()
@ -92,35 +90,3 @@ class FixMap(basefix.BaseFix):
new = Call(Name("list"), [new])
new.set_prefix(node.get_prefix())
return new
P0 = """for_stmt< 'for' any 'in' node=any ':' any* >
| comp_for< 'for' any 'in' node=any any* >
"""
p0 = patcomp.compile_pattern(P0)
P1 = """
power<
( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
'any' | 'all' | (any* trailer< '.' 'join' >) )
trailer< '(' node=any ')' >
any*
>
"""
p1 = patcomp.compile_pattern(P1)
P2 = """
power<
'sorted'
trailer< '(' arglist<node=any any*> ')' >
any*
>
"""
p2 = patcomp.compile_pattern(P2)
def in_special_context(node):
patterns = [p0, p1, p2]
for pattern, parent in zip(patterns, attr_chain(node, "parent")):
results = {}
if pattern.match(parent, results) and results["node"] is node:
return True
return False

6
Lib/lib2to3/fixes/fix_ne.py
View File

@ -17,6 +17,6 @@ class FixNe(basefix.BaseFix):
return node.type == token.NOTEQUAL and node.value == "<>"
def transform(self, node, results):
new = pytree.Leaf(token.NOTEQUAL, "!=")
new.set_prefix(node.get_prefix())
return new
new = pytree.Leaf(token.NOTEQUAL, "!=")
new.set_prefix(node.get_prefix())
return new

12
Lib/lib2to3/fixes/fix_next.py
View File

@ -47,12 +47,12 @@ class FixNext(basefix.BaseFix):
mod = results.get("mod")
if base:
if self.shadowed_next:
attr.replace(Name("__next__", prefix=attr.get_prefix()))
else:
base = [n.clone() for n in base]
base[0].set_prefix("")
node.replace(Call(Name("next", prefix=node.get_prefix()), base))
if self.shadowed_next:
attr.replace(Name("__next__", prefix=attr.get_prefix()))
else:
base = [n.clone() for n in base]
base[0].set_prefix("")
node.replace(Call(Name("next", prefix=node.get_prefix()), base))
elif name:
n = Name("__next__", prefix=name.get_prefix())
name.replace(n)

2
Lib/lib2to3/fixes/fix_print.py
View File

@ -5,7 +5,7 @@
Change:
'print' into 'print()'
'print ...' into 'print(...)'
'print ...' into 'print(...)'
'print ... ,' into 'print(..., end=" ")'
'print >>x, ...' into 'print(..., file=x)'
"""

14
Lib/lib2to3/fixes/fix_raise.py
View File

@ -12,9 +12,9 @@ raise "foo", V, T -> warns about string exceptions
CAVEATS:
1) "raise E, V" will be incorrectly translated if V is an exception
instance. The correct Python 3 idiom is
raise E from V
but since we can't detect instance-hood by syntax alone and since
any client code would have to be changed as well, we don't automate
this.
@ -48,11 +48,11 @@ class FixRaise(basefix.BaseFix):
# Since Python 3 will not support this, we recurse down any tuple
# literals, always taking the first element.
if is_tuple(exc):
while is_tuple(exc):
# exc.children[1:-1] is the unparenthesized tuple
# exc.children[1].children[0] is the first element of the tuple
exc = exc.children[1].children[0].clone()
exc.set_prefix(" ")
while is_tuple(exc):
# exc.children[1:-1] is the unparenthesized tuple
# exc.children[1].children[0] is the first element of the tuple
exc = exc.children[1].children[0].clone()
exc.set_prefix(" ")
if "val" not in results:
# One-argument raise

3
Lib/lib2to3/fixes/fix_renames.py
View File

@ -63,8 +63,7 @@ class FixRenames(basefix.BaseFix):
attr_name = results.get("attr_name")
#bare_name = results.get("bare_name")
#import_mod = results.get("module")
if mod_name and attr_name:
new_attr = LOOKUP[(mod_name.value, attr_name.value)]
attr_name.replace(Name(new_attr, prefix=attr_name.get_prefix()))

8
Lib/lib2to3/fixes/fix_repr.py
View File

@ -15,8 +15,8 @@ class FixRepr(basefix.BaseFix):
"""
def transform(self, node, results):
expr = results["expr"].clone()
expr = results["expr"].clone()
if expr.type == self.syms.testlist1:
expr = self.parenthesize(expr)
return Call(Name("repr"), [expr], prefix=node.get_prefix())
if expr.type == self.syms.testlist1:
expr = self.parenthesize(expr)
return Call(Name("repr"), [expr], prefix=node.get_prefix())

4
Lib/lib2to3/fixes/fix_tuple_params.py
View File

@ -2,7 +2,7 @@
def func(((a, b), c), d):
...
->
def func(x, d):
@ -10,7 +10,7 @@ def func(x, d):
...
It will also support lambdas:
lambda (x, y): x + y -> lambda t: t[0] + t[1]
# The parens are a syntax error in Python 3

34
Lib/lib2to3/fixes/fix_unicode.py
View File

@ -8,21 +8,21 @@ from .import basefix
class FixUnicode(basefix.BaseFix):
PATTERN = "STRING | NAME<'unicode' | 'unichr'>"
PATTERN = "STRING | NAME<'unicode' | 'unichr'>"
def transform(self, node, results):
if node.type == token.NAME:
if node.value == "unicode":
new = node.clone()
new.value = "str"
return new
if node.value == "unichr":
new = node.clone()
new.value = "chr"
return new
# XXX Warn when __unicode__ found?
elif node.type == token.STRING:
if re.match(r"[uU][rR]?[\'\"]", node.value):
new = node.clone()
new.value = new.value[1:]
return new
def transform(self, node, results):
if node.type == token.NAME:
if node.value == "unicode":
new = node.clone()
new.value = "str"
return new
if node.value == "unichr":
new = node.clone()
new.value = "chr"
return new
# XXX Warn when __unicode__ found?
elif node.type == token.STRING:
if re.match(r"[uU][rR]?[\'\"]", node.value):
new = node.clone()
new.value = new.value[1:]
return new

44
Lib/lib2to3/fixes/fix_ws_comma.py
View File

@ -11,29 +11,29 @@ from .import basefix
class FixWsComma(basefix.BaseFix):
explicit = True # The user must ask for this fixers
explicit = True # The user must ask for this fixers
PATTERN = """
any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
"""
PATTERN = """
any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
"""
COMMA = pytree.Leaf(token.COMMA, ",")
COLON = pytree.Leaf(token.COLON, ":")
SEPS = (COMMA, COLON)
COMMA = pytree.Leaf(token.COMMA, ",")
COLON = pytree.Leaf(token.COLON, ":")
SEPS = (COMMA, COLON)
def transform(self, node, results):
new = node.clone()
comma = False
for child in new.children:
if child in self.SEPS:
prefix = child.get_prefix()
if prefix.isspace() and "\n" not in prefix:
child.set_prefix("")
comma = True
else:
if comma:
prefix = child.get_prefix()
if not prefix:
child.set_prefix(" ")
def transform(self, node, results):
new = node.clone()
comma = False
return new
for child in new.children:
if child in self.SEPS:
prefix = child.get_prefix()
if prefix.isspace() and "\n" not in prefix:
child.set_prefix("")
comma = True
else:
if comma:
prefix = child.get_prefix()
if not prefix:
child.set_prefix(" ")
comma = False
return new

43
Lib/lib2to3/fixes/fix_zip.py Normal file
View File

@ -0,0 +1,43 @@
"""
Fixer that changes zip(seq0, seq1, ...) into list(zip(seq0, seq1, ...)
unless there exists a 'from future_builtins import zip' statement in the
top-level namespace.
We avoid the transformation if the zip() call is directly contained in
iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:.
"""
# Local imports
from . import basefix
from .util import Name, Call, does_tree_import, in_special_context
class FixZip(basefix.BaseFix):
PATTERN = """
power< 'zip' args=trailer< '(' [any] ')' >
>
"""
def start_tree(self, *args):
super(FixZip, self).start_tree(*args)
self._future_zip_found = None
def has_future_zip(self, node):
if self._future_zip_found is not None:
return self._future_zip_found
self._future_zip_found = does_tree_import('future_builtins', 'zip', node)
return self._future_zip_found
def transform(self, node, results):
if self.has_future_zip(node):
# If a future zip has been imported for this file, we won't
# be making any modifications
return
if in_special_context(node):
return None
new = node.clone()
new.set_prefix("")
new = Call(Name("list"), [new])
new.set_prefix(node.get_prefix())
return new

49
Lib/lib2to3/fixes/util.py
View File

@ -5,6 +5,7 @@
from ..pgen2 import token
from ..pytree import Leaf, Node
from ..pygram import python_symbols as syms
from .. import patcomp
###########################################################
@ -163,15 +164,15 @@ except NameError:
def attr_chain(obj, attr):
"""Follow an attribute chain.
If you have a chain of objects where a.foo -> b, b.foo-> c, etc,
use this to iterate over all objects in the chain. Iteration is
terminated by getattr(x, attr) is None.
Args:
obj: the starting object
attr: the name of the chaining attribute
Yields:
Each successive object in the chain.
"""
@ -180,6 +181,44 @@ def attr_chain(obj, attr):
yield next
next = getattr(next, attr)
p0 = """for_stmt< 'for' any 'in' node=any ':' any* >
| comp_for< 'for' any 'in' node=any any* >
"""
p1 = """
power<
( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
'any' | 'all' | (any* trailer< '.' 'join' >) )
trailer< '(' node=any ')' >
any*
>
"""
p2 = """
power<
'sorted'
trailer< '(' arglist<node=any any*> ')' >
any*
>
"""
pats_built = False
def in_special_context(node):
""" Returns true if node is in an environment where all that is required
of it is being itterable (ie, it doesn't matter if it returns a list
or an itterator).
See test_map_nochange in test_fixers.py for some examples and tests.
"""
global p0, p1, p2, pats_built
if not pats_built:
p1 = patcomp.compile_pattern(p1)
p0 = patcomp.compile_pattern(p0)
p2 = patcomp.compile_pattern(p2)
pats_built = True
patterns = [p0, p1, p2]
for pattern, parent in zip(patterns, attr_chain(node, "parent")):
results = {}
if pattern.match(parent, results) and results["node"] is node:
return True
return False
###########################################################
### The following functions are to find bindings in a suite
###########################################################
@ -240,8 +279,8 @@ def find_binding(name, node, package=None):
elif child.type == syms.simple_stmt:
ret = find_binding(name, child, package)
elif child.type == syms.expr_stmt:
if _find(name, child.children[0]):
ret = child
if _find(name, child.children[0]):
ret = child
if ret:
if not package:

18
Lib/lib2to3/refactor.py
View File

@ -118,7 +118,7 @@ class RefactoringTool(object):
def get_fixers(self):
"""Inspects the options to load the requested patterns and handlers.
Returns:
(pre_order, post_order), where pre_order is the list of fixers that
want a pre-order AST traversal, and post_order is the list that want
@ -232,11 +232,11 @@ class RefactoringTool(object):
def refactor_string(self, data, name):
"""Refactor a given input string.
Args:
data: a string holding the code to be refactored.
name: a human-readable name for use in error/log messages.
Returns:
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
@ -274,12 +274,12 @@ class RefactoringTool(object):
def refactor_tree(self, tree, name):
"""Refactors a parse tree (modifying the tree in place).
Args:
tree: a pytree.Node instance representing the root of the tree
to be refactored.
name: a human-readable name for this tree.
Returns:
True if the tree was modified, False otherwise.
"""
@ -296,13 +296,13 @@ class RefactoringTool(object):
def traverse_by(self, fixers, traversal):
"""Traverse an AST, applying a set of fixers to each node.
This is a helper method for refactor_tree().
Args:
fixers: a list of fixer instances.
traversal: a generator that yields AST nodes.
Returns:
None
"""
@ -523,4 +523,4 @@ def diff_texts(a, b, filename):
if __name__ == "__main__":
sys.exit(main())
sys.exit(main())

4
Lib/lib2to3/tests/benchmark.py
View File

@ -27,10 +27,10 @@ class Options:
setattr(self, k, v)
self.verbose = False
def dummy_transform(*args, **kwargs):
pass
### Collect list of modules to match against
###############################################################################
files = []

63
Lib/lib2to3/tests/test_fixers.py
View File

@ -2493,6 +2493,69 @@ class Test_map(FixerTestCase):
a = "from future_builtins import *; map(f, 'ham')"
self.unchanged(a)
class Test_zip(FixerTestCase):
fixer = "zip"
def check(self, b, a):
self.unchanged("from future_builtins import zip; " + b, a)
FixerTestCase.check(self, b, a)
def test_zip_basic(self):
b = """x = zip(a, b, c)"""
a = """x = list(zip(a, b, c))"""
self.check(b, a)
b = """x = len(zip(a, b))"""
a = """x = len(list(zip(a, b)))"""
self.check(b, a)
def test_zip_nochange(self):
a = """b.join(zip(a, b))"""
self.unchanged(a)
a = """(a + foo(5)).join(zip(a, b))"""
self.unchanged(a)
a = """iter(zip(a, b))"""
self.unchanged(a)
a = """list(zip(a, b))"""
self.unchanged(a)
a = """list(zip(a, b))[0]"""
self.unchanged(a)
a = """set(zip(a, b))"""
self.unchanged(a)
a = """set(zip(a, b)).pop()"""
self.unchanged(a)
a = """tuple(zip(a, b))"""
self.unchanged(a)
a = """any(zip(a, b))"""
self.unchanged(a)
a = """all(zip(a, b))"""
self.unchanged(a)
a = """sum(zip(a, b))"""
self.unchanged(a)
a = """sorted(zip(a, b))"""
self.unchanged(a)
a = """sorted(zip(a, b), key=blah)"""
self.unchanged(a)
a = """sorted(zip(a, b), key=blah)[0]"""
self.unchanged(a)
a = """for i in zip(a, b): pass"""
self.unchanged(a)
a = """[x for x in zip(a, b)]"""
self.unchanged(a)
a = """(x for x in zip(a, b))"""
self.unchanged(a)
def test_future_builtins(self):
a = "from future_builtins import spam, zip, eggs; zip(a, b)"
self.unchanged(a)
b = """from future_builtins import spam, eggs; x = zip(a, b)"""
a = """from future_builtins import spam, eggs; x = list(zip(a, b))"""
self.check(b, a)
a = "from future_builtins import *; zip(a, b)"
self.unchanged(a)
class Test_standarderror(FixerTestCase):
fixer = "standarderror"

2
Lib/lib2to3/tests/test_parser.py
View File

@ -109,7 +109,7 @@ class TestExcept(GrammarTest):
self.validate(s)
# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
class TestSetLiteral(GrammarTest):
def test_1(self):
self.validate("""x = {'one'}""")

4
Lib/lib2to3/tests/test_pytree.py
View File

@ -407,10 +407,10 @@ class TestPatterns(support.TestCase):
pe = pytree.LeafPattern(1, "e", "pe")
pf = pytree.LeafPattern(1, "f", "pf")
pw = pytree.WildcardPattern([[pa, pb, pc], [pd, pe],
[pa, pb], [pc, pd], [pe, pf]],
[pa, pb], [pc, pd], [pe, pf]],
min=1, max=4, name="pw")
self.assertEqual([x[0] for x in pw.generate_matches(leaves)],
[3, 5, 2, 4, 6])
[3, 5, 2, 4, 6])
pr = pytree.NodePattern(type=1000, content=[pw], name="pr")
matches = list(pytree.generate_matches([pr], [root]))
self.assertEqual(len(matches), 1)