Merged revisions 67806 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

................
  r67806 | benjamin.peterson | 2008-12-15 21:35:28 -0600 (Mon, 15 Dec 2008) | 111 lines

  Merged revisions 67427,67431,67433,67435,67630,67652,67656-67657,67674-67675,67678-67679,67705-67706,67716,67723,67765-67771,67774,67776,67778 via svnmerge from
  svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

  ........
    r67427 | benjamin.peterson | 2008-11-28 16:07:41 -0600 (Fri, 28 Nov 2008) | 1 line

    fix spelling in comment
  ........
    r67431 | benjamin.peterson | 2008-11-28 17:14:08 -0600 (Fri, 28 Nov 2008) | 1 line

    add a scripts directory; move things to it
  ........
    r67433 | benjamin.peterson | 2008-11-28 17:18:48 -0600 (Fri, 28 Nov 2008) | 1 line

    run svneol.py
  ........
    r67435 | benjamin.peterson | 2008-11-28 17:25:03 -0600 (Fri, 28 Nov 2008) | 1 line

    rename pre/post_order_mapping to pre/post_order_heads
  ........
    r67630 | alexandre.vassalotti | 2008-12-06 21:51:56 -0600 (Sat, 06 Dec 2008) | 2 lines

    Fix typo in the urllib2.HTTPDigestAuthHandler fixer.
  ........
    r67652 | armin.ronacher | 2008-12-07 15:39:43 -0600 (Sun, 07 Dec 2008) | 5 lines

    Added a fixer that cleans up a tuple argument to isinstance after the tokens
    in it were fixed.  This is mainly used to remove double occurrences of
    tokens as a leftover of the long -> int / unicode -> str conversion.
  ........
    r67656 | armin.ronacher | 2008-12-07 16:54:16 -0600 (Sun, 07 Dec 2008) | 3 lines

    Added missing copyright fo 2to3 fix_isinstance.
  ........
    r67657 | armin.ronacher | 2008-12-07 18:29:35 -0600 (Sun, 07 Dec 2008) | 3 lines

    2to3: intern and reduce fixes now add the imports if missing.  Because that is a common task the fixer_util module now has a function "touch_import" that adds imports if missing.
  ........
    r67674 | benjamin.peterson | 2008-12-08 19:58:11 -0600 (Mon, 08 Dec 2008) | 1 line

    copy permission bits when making backup files #4602
  ........
    r67675 | benjamin.peterson | 2008-12-08 19:59:11 -0600 (Mon, 08 Dec 2008) | 1 line

    add forgotten import
  ........
    r67678 | benjamin.peterson | 2008-12-08 20:08:30 -0600 (Mon, 08 Dec 2008) | 1 line

    fix #4602 for real
  ........
    r67679 | armin.ronacher | 2008-12-09 00:54:03 -0600 (Tue, 09 Dec 2008) | 3 lines

    Removed redudant code from the 2to3 long fixer.  This fixes #4590.
  ........
    r67705 | benjamin.peterson | 2008-12-11 13:04:08 -0600 (Thu, 11 Dec 2008) | 1 line

    put trailers after a range call after the list()
  ........
    r67706 | benjamin.peterson | 2008-12-11 13:17:57 -0600 (Thu, 11 Dec 2008) | 1 line

    add html related modules to the fix_imports mapping
  ........
    r67716 | benjamin.peterson | 2008-12-11 22:16:47 -0600 (Thu, 11 Dec 2008) | 1 line

    consolidate tests
  ........
    r67723 | benjamin.peterson | 2008-12-12 19:49:31 -0600 (Fri, 12 Dec 2008) | 1 line

    fix name
  ........
    r67765 | benjamin.peterson | 2008-12-14 14:05:05 -0600 (Sun, 14 Dec 2008) | 1 line

    run fix_isinstance after fix_long and fix_unicode
  ........
    r67766 | benjamin.peterson | 2008-12-14 14:13:05 -0600 (Sun, 14 Dec 2008) | 1 line

    use run_order instead of order
  ........
    r67767 | benjamin.peterson | 2008-12-14 14:28:12 -0600 (Sun, 14 Dec 2008) | 1 line

    don't retain parenthesis if there is only one item left
  ........
    r67768 | benjamin.peterson | 2008-12-14 14:32:30 -0600 (Sun, 14 Dec 2008) | 1 line

    use insert_child()
  ........
    r67769 | benjamin.peterson | 2008-12-14 14:59:10 -0600 (Sun, 14 Dec 2008) | 1 line

    parenthesize doesn't belong in pygram or FixerBase
  ........
    r67770 | alexandre.vassalotti | 2008-12-14 15:15:36 -0600 (Sun, 14 Dec 2008) | 2 lines

    Fix typo: html.paser -> html.parser.
  ........
    r67771 | benjamin.peterson | 2008-12-14 15:22:09 -0600 (Sun, 14 Dec 2008) | 1 line

    altering .children needs to call changed()
  ........
    r67774 | benjamin.peterson | 2008-12-14 15:55:38 -0600 (Sun, 14 Dec 2008) | 1 line

    employ an evil hack to fix multiple names in the same import statement
  ........
    r67776 | benjamin.peterson | 2008-12-14 16:22:38 -0600 (Sun, 14 Dec 2008) | 1 line

    make a common mixin class for Test_imports and friends
  ........
    r67778 | alexandre.vassalotti | 2008-12-14 17:48:20 -0600 (Sun, 14 Dec 2008) | 2 lines

    Make fix_imports refactor multiple imports as.
  ........
................
This commit is contained in:
Benjamin Peterson 2008-12-16 03:49:59 +00:00
parent 360d8cfc9d
commit c3224e2ca8
20 changed files with 334 additions and 158 deletions

View File

@ -94,10 +94,6 @@ class BaseFix(object):
""" """
raise NotImplementedError() raise NotImplementedError()
def parenthesize(self, node):
"""Wrapper around pygram.parenthesize()."""
return pygram.parenthesize(node)
def new_name(self, template="xxx_todo_changeme"): def new_name(self, template="xxx_todo_changeme"):
"""Return a string suitable for use as an identifier """Return a string suitable for use as an identifier

View File

@ -158,6 +158,9 @@ def is_list(node):
### Misc ### Misc
########################################################### ###########################################################
def parenthesize(node):
return Node(syms.atom, [LParen(), node, RParen()])
consuming_calls = set(["sorted", "list", "set", "any", "all", "tuple", "sum", consuming_calls = set(["sorted", "list", "set", "any", "all", "tuple", "sum",
"min", "max"]) "min", "max"])
@ -232,20 +235,77 @@ def make_suite(node):
suite.parent = parent suite.parent = parent
return suite return suite
def does_tree_import(package, name, node): def find_root(node):
""" Returns true if name is imported from package at the """Find the top level namespace."""
top level of the tree which node belongs to.
To cover the case of an import like 'import foo', use
Null for the package and 'foo' for the name. """
# Scamper up to the top level namespace # Scamper up to the top level namespace
while node.type != syms.file_input: while node.type != syms.file_input:
assert node.parent, "Tree is insane! root found before "\ assert node.parent, "Tree is insane! root found before "\
"file_input node was found." "file_input node was found."
node = node.parent node = node.parent
return node
binding = find_binding(name, node, package) def does_tree_import(package, name, node):
""" Returns true if name is imported from package at the
top level of the tree which node belongs to.
To cover the case of an import like 'import foo', use
None for the package and 'foo' for the name. """
binding = find_binding(name, find_root(node), package)
return bool(binding) return bool(binding)
def is_import(node):
"""Returns true if the node is an import statement."""
return node.type in (syms.import_name, syms.import_from)
def touch_import(package, name, node):
""" Works like `does_tree_import` but adds an import statement
if it was not imported. """
def is_import_stmt(node):
return node.type == syms.simple_stmt and node.children and \
is_import(node.children[0])
root = find_root(node)
if does_tree_import(package, name, root):
return
add_newline_before = False
# figure out where to insert the new import. First try to find
# the first import and then skip to the last one.
insert_pos = offset = 0
for idx, node in enumerate(root.children):
if not is_import_stmt(node):
continue
for offset, node2 in enumerate(root.children[idx:]):
if not is_import_stmt(node2):
break
insert_pos = idx + offset
break
# if there are no imports where we can insert, find the docstring.
# if that also fails, we stick to the beginning of the file
if insert_pos == 0:
for idx, node in enumerate(root.children):
if node.type == syms.simple_stmt and node.children and \
node.children[0].type == token.STRING:
insert_pos = idx + 1
add_newline_before
break
if package is None:
import_ = Node(syms.import_name, [
Leaf(token.NAME, 'import'),
Leaf(token.NAME, name, prefix=' ')
])
else:
import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')])
children = [import_, Newline()]
if add_newline_before:
children.insert(0, Newline())
root.insert_child(insert_pos, Node(syms.simple_stmt, children))
_def_syms = set([syms.classdef, syms.funcdef]) _def_syms = set([syms.classdef, syms.funcdef])
def find_binding(name, node, package=None): def find_binding(name, node, package=None):
""" Returns the node which binds variable name, otherwise None. """ Returns the node which binds variable name, otherwise None.
@ -285,7 +345,7 @@ def find_binding(name, node, package=None):
if ret: if ret:
if not package: if not package:
return ret return ret
if ret.type in (syms.import_name, syms.import_from): if is_import(ret):
return ret return ret
return None return None

View File

@ -9,7 +9,7 @@ This converts apply(func, v, k) into (func)(*v, **k)."""
from .. import pytree from .. import pytree
from ..pgen2 import token from ..pgen2 import token
from .. import fixer_base from .. import fixer_base
from ..fixer_util import Call, Comma from ..fixer_util import Call, Comma, parenthesize
class FixApply(fixer_base.BaseFix): class FixApply(fixer_base.BaseFix):
@ -39,7 +39,7 @@ class FixApply(fixer_base.BaseFix):
(func.type != syms.power or (func.type != syms.power or
func.children[-2].type == token.DOUBLESTAR)): func.children[-2].type == token.DOUBLESTAR)):
# Need to parenthesize # Need to parenthesize
func = self.parenthesize(func) func = parenthesize(func)
func.set_prefix("") func.set_prefix("")
args = args.clone() args = args.clone()
args.set_prefix("") args.set_prefix("")

View File

@ -33,7 +33,7 @@ CAVEATS:
from .. import pytree from .. import pytree
from ..pgen2 import token from ..pgen2 import token
from .. import fixer_base from .. import fixer_base
from ..fixer_util import Name from ..fixer_util import Name, parenthesize
class FixHasKey(fixer_base.BaseFix): class FixHasKey(fixer_base.BaseFix):
@ -86,7 +86,7 @@ class FixHasKey(fixer_base.BaseFix):
after = [n.clone() for n in after] after = [n.clone() for n in after]
if arg.type in (syms.comparison, syms.not_test, syms.and_test, if arg.type in (syms.comparison, syms.not_test, syms.and_test,
syms.or_test, syms.test, syms.lambdef, syms.argument): syms.or_test, syms.test, syms.lambdef, syms.argument):
arg = self.parenthesize(arg) arg = parenthesize(arg)
if len(before) == 1: if len(before) == 1:
before = before[0] before = before[0]
else: else:
@ -98,12 +98,12 @@ class FixHasKey(fixer_base.BaseFix):
n_op = pytree.Node(syms.comp_op, (n_not, n_op)) n_op = pytree.Node(syms.comp_op, (n_not, n_op))
new = pytree.Node(syms.comparison, (arg, n_op, before)) new = pytree.Node(syms.comparison, (arg, n_op, before))
if after: if after:
new = self.parenthesize(new) new = parenthesize(new)
new = pytree.Node(syms.power, (new,) + tuple(after)) new = pytree.Node(syms.power, (new,) + tuple(after))
if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr, if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr,
syms.and_expr, syms.shift_expr, syms.and_expr, syms.shift_expr,
syms.arith_expr, syms.term, syms.arith_expr, syms.term,
syms.factor, syms.power): syms.factor, syms.power):
new = self.parenthesize(new) new = parenthesize(new)
new.set_prefix(prefix) new.set_prefix(prefix)
return new return new

View File

@ -42,6 +42,8 @@ MAPPING = {'StringIO': 'io',
'DocXMLRPCServer': 'xmlrpc.server', 'DocXMLRPCServer': 'xmlrpc.server',
'SimpleXMLRPCServer': 'xmlrpc.server', 'SimpleXMLRPCServer': 'xmlrpc.server',
'httplib': 'http.client', 'httplib': 'http.client',
'htmlentitydefs' : 'html.entities',
'HTMLParser' : 'html.parser',
'Cookie': 'http.cookies', 'Cookie': 'http.cookies',
'cookielib': 'http.cookiejar', 'cookielib': 'http.cookiejar',
'BaseHTTPServer': 'http.server', 'BaseHTTPServer': 'http.server',
@ -64,16 +66,17 @@ def build_pattern(mapping=MAPPING):
mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) mod_list = ' | '.join(["module_name='%s'" % key for key in mapping])
bare_names = alternates(mapping.keys()) bare_names = alternates(mapping.keys())
yield """name_import=import_name< 'import' ((%s) yield """name_import=import_name< 'import' ((%s) |
| dotted_as_names< any* (%s) any* >) > multiple_imports=dotted_as_names< any* (%s) any* >) >
""" % (mod_list, mod_list) """ % (mod_list, mod_list)
yield """import_from< 'from' (%s) 'import' ['('] yield """import_from< 'from' (%s) 'import' ['(']
( any | import_as_name< any 'as' any > | ( any | import_as_name< any 'as' any > |
import_as_names< any* >) [')'] > import_as_names< any* >) [')'] >
""" % mod_list """ % mod_list
yield """import_name< 'import' yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > |
dotted_as_name< (%s) 'as' any > > multiple_imports=dotted_as_names<
""" % mod_list any* dotted_as_name< (%s) 'as' any > any* >) >
""" % (mod_list, mod_list)
# Find usages of module members in code e.g. thread.foo(bar) # Find usages of module members in code e.g. thread.foo(bar)
yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names
@ -100,8 +103,8 @@ class FixImports(fixer_base.BaseFix):
match = super(FixImports, self).match match = super(FixImports, self).match
results = match(node) results = match(node)
if results: if results:
# Module usage could be in the trailier of an attribute lookup, so # Module usage could be in the trailer of an attribute lookup, so we
# we might have nested matches when "bare_with_attr" is present. # might have nested matches when "bare_with_attr" is present.
if "bare_with_attr" not in results and \ if "bare_with_attr" not in results and \
any([match(obj) for obj in attr_chain(node, "parent")]): any([match(obj) for obj in attr_chain(node, "parent")]):
return False return False
@ -116,11 +119,21 @@ class FixImports(fixer_base.BaseFix):
import_mod = results.get("module_name") import_mod = results.get("module_name")
if import_mod: if import_mod:
new_name = self.mapping[(import_mod or mod_name).value] new_name = self.mapping[(import_mod or mod_name).value]
import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
if "name_import" in results: if "name_import" in results:
# If it's not a "from x import x, y" or "import x as y" import, # If it's not a "from x import x, y" or "import x as y" import,
# marked its usage to be replaced. # marked its usage to be replaced.
self.replace[import_mod.value] = new_name self.replace[import_mod.value] = new_name
import_mod.replace(Name(new_name, prefix=import_mod.get_prefix())) if "multiple_imports" in results:
# This is a nasty hack to fix multiple imports on a
# line (e.g., "import StringIO, urlparse"). The problem is that I
# can't figure out an easy way to make a pattern recognize the
# keys of MAPPING randomly sprinkled in an import statement.
while True:
results = self.match(node)
if not results:
break
self.transform(node, results)
else: else:
# Replace usage of the module. # Replace usage of the module.
bare_name = results["bare_with_attr"][0] bare_name = results["bare_with_attr"][0]

View File

@ -11,6 +11,6 @@ MAPPING = {
class FixImports2(fix_imports.FixImports): class FixImports2(fix_imports.FixImports):
order = "post" run_order = 6
mapping = MAPPING mapping = MAPPING

View File

@ -8,7 +8,7 @@ intern(s) -> sys.intern(s)"""
# Local imports # Local imports
from .. import pytree from .. import pytree
from .. import fixer_base from .. import fixer_base
from ..fixer_util import Name, Attr from ..fixer_util import Name, Attr, touch_import
class FixIntern(fixer_base.BaseFix): class FixIntern(fixer_base.BaseFix):
@ -40,4 +40,5 @@ class FixIntern(fixer_base.BaseFix):
newarglist, newarglist,
results["rpar"].clone()])] + after) results["rpar"].clone()])] + after)
new.set_prefix(node.get_prefix()) new.set_prefix(node.get_prefix())
touch_import(None, 'sys', node)
return new return new

View File

@ -0,0 +1,52 @@
# Copyright 2008 Armin Ronacher.
# Licensed to PSF under a Contributor Agreement.
"""Fixer that cleans up a tuple argument to isinstance after the tokens
in it were fixed. This is mainly used to remove double occurrences of
tokens as a leftover of the long -> int / unicode -> str conversion.
eg. isinstance(x, (int, long)) -> isinstance(x, (int, int))
-> isinstance(x, int)
"""
from .. import fixer_base
from ..fixer_util import token
class FixIsinstance(fixer_base.BaseFix):
PATTERN = """
power<
'isinstance'
trailer< '(' arglist< any ',' atom< '('
args=testlist_gexp< any+ >
')' > > ')' >
>
"""
run_order = 6
def transform(self, node, results):
names_inserted = set()
testlist = results["args"]
args = testlist.children
new_args = []
iterator = enumerate(args)
for idx, arg in iterator:
if arg.type == token.NAME and arg.value in names_inserted:
if idx < len(args) - 1 and args[idx + 1].type == token.COMMA:
iterator.next()
continue
else:
new_args.append(arg)
if arg.type == token.NAME:
names_inserted.add(arg.value)
if new_args and new_args[-1].type == token.COMMA:
del new_args[-1]
if len(new_args) == 1:
atom = testlist.parent
new_args[0].set_prefix(atom.get_prefix())
atom.replace(new_args[0])
else:
args[:] = new_args
node.changed()

View File

@ -2,8 +2,6 @@
# Licensed to PSF under a Contributor Agreement. # Licensed to PSF under a Contributor Agreement.
"""Fixer that turns 'long' into 'int' everywhere. """Fixer that turns 'long' into 'int' everywhere.
This also strips the trailing 'L' or 'l' from long loterals.
""" """
# Local imports # Local imports
@ -14,22 +12,13 @@ from ..fixer_util import Name, Number
class FixLong(fixer_base.BaseFix): class FixLong(fixer_base.BaseFix):
PATTERN = """ PATTERN = "'long'"
(long_type = 'long' | number = NUMBER)
"""
static_long = Name("long") static_long = Name("long")
static_int = Name("int") static_int = Name("int")
def transform(self, node, results): def transform(self, node, results):
long_type = results.get("long_type") assert node == self.static_long, node
number = results.get("number") new = self.static_int.clone()
new = None new.set_prefix(node.get_prefix())
if long_type: return new
assert node == self.static_long, node
new = self.static_int.clone()
if number and node.value[-1] in ("l", "L"):
new = Number(node.value[:-1])
if new is not None:
new.set_prefix(node.get_prefix())
return new

View File

@ -0,0 +1,33 @@
# Copyright 2008 Armin Ronacher.
# Licensed to PSF under a Contributor Agreement.
"""Fixer for reduce().
Makes sure reduce() is imported from the functools module if reduce is
used in that module.
"""
from .. import pytree
from .. import fixer_base
from ..fixer_util import Name, Attr, touch_import
class FixReduce(fixer_base.BaseFix):
PATTERN = """
power< 'reduce'
trailer< '('
arglist< (
(not(argument<any '=' any>) any ','
not(argument<any '=' any>) any) |
(not(argument<any '=' any>) any ','
not(argument<any '=' any>) any ','
not(argument<any '=' any>) any)
) >
')' >
>
"""
def transform(self, node, results):
touch_import('functools', 'reduce', node)

View File

@ -5,7 +5,7 @@
# Local imports # Local imports
from .. import fixer_base from .. import fixer_base
from ..fixer_util import Call, Name from ..fixer_util import Call, Name, parenthesize
class FixRepr(fixer_base.BaseFix): class FixRepr(fixer_base.BaseFix):
@ -18,5 +18,5 @@ class FixRepr(fixer_base.BaseFix):
expr = results["expr"].clone() expr = results["expr"].clone()
if expr.type == self.syms.testlist1: if expr.type == self.syms.testlist1:
expr = self.parenthesize(expr) expr = parenthesize(expr)
return Call(Name("repr"), [expr], prefix=node.get_prefix()) return Call(Name("repr"), [expr], prefix=node.get_prefix())

View File

@ -29,7 +29,7 @@ MAPPING = {'urllib': [
'AbstractBasicAuthHandler', 'AbstractBasicAuthHandler',
'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler',
'AbstractDigestAuthHandler', 'AbstractDigestAuthHandler',
'HTTPDigestAuthHander', 'ProxyDigestAuthHandler', 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler',
'HTTPHandler', 'HTTPSHandler', 'FileHandler', 'HTTPHandler', 'HTTPSHandler', 'FileHandler',
'FTPHandler', 'CacheFTPHandler', 'FTPHandler', 'CacheFTPHandler',
'UnknownHandler']), 'UnknownHandler']),

View File

@ -12,7 +12,9 @@ from .. import patcomp
class FixXrange(fixer_base.BaseFix): class FixXrange(fixer_base.BaseFix):
PATTERN = """ PATTERN = """
power< (name='range'|name='xrange') trailer< '(' [any] ')' > any* > power<
(name='range'|name='xrange') trailer< '(' args=any ')' >
rest=any* >
""" """
def transform(self, node, results): def transform(self, node, results):
@ -30,11 +32,14 @@ class FixXrange(fixer_base.BaseFix):
def transform_range(self, node, results): def transform_range(self, node, results):
if not self.in_special_context(node): if not self.in_special_context(node):
arg = node.clone() range_call = Call(Name("range"), [results["args"].clone()])
arg.set_prefix("") # Encase the range call in list().
call = Call(Name("list"), [arg]) list_call = Call(Name("list"), [range_call],
call.set_prefix(node.get_prefix()) prefix=node.get_prefix())
return call # Put things that were after the range() call after the list call.
for n in results["rest"]:
list_call.append_child(n)
return list_call
return node return node
P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" P1 = "power< func=NAME trailer< '(' node=any ')' > any* >"

View File

@ -5,6 +5,7 @@ Main program for 2to3.
import sys import sys
import os import os
import logging import logging
import shutil
import optparse import optparse
from . import refactor from . import refactor
@ -39,6 +40,7 @@ class StdoutRefactoringTool(refactor.RefactoringTool):
# Actually write the new file # Actually write the new file
super(StdoutRefactoringTool, self).write_file(new_text, super(StdoutRefactoringTool, self).write_file(new_text,
filename, old_text) filename, old_text)
shutil.copymode(filename, backup)
def print_output(self, lines): def print_output(self, lines):
for line in lines: for line in lines:
@ -56,7 +58,7 @@ def main(fixer_pkg, args=None):
Returns a suggested exit status (0, 1, 2). Returns a suggested exit status (0, 1, 2).
""" """
# Set up option parser # Set up option parser
parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...") parser = optparse.OptionParser(usage="2to3 [options] file|dir ...")
parser.add_option("-d", "--doctests_only", action="store_true", parser.add_option("-d", "--doctests_only", action="store_true",
help="Fix up doctests only") help="Fix up doctests only")
parser.add_option("-f", "--fix", action="append", default=[], parser.add_option("-f", "--fix", action="append", default=[],

View File

@ -29,10 +29,3 @@ class Symbols(object):
python_grammar = driver.load_grammar(_GRAMMAR_FILE) python_grammar = driver.load_grammar(_GRAMMAR_FILE)
python_symbols = Symbols(python_grammar) python_symbols = Symbols(python_grammar)
def parenthesize(node):
return pytree.Node(python_symbols.atom,
(pytree.Leaf(token.LPAR, "("),
node,
pytree.Leaf(token.RPAR, ")")))

View File

@ -279,18 +279,21 @@ class Node(Base):
child.parent = self child.parent = self
self.children[i].parent = None self.children[i].parent = None
self.children[i] = child self.children[i] = child
self.changed()
def insert_child(self, i, child): def insert_child(self, i, child):
"""Equivalent to 'node.children.insert(i, child)'. This method also """Equivalent to 'node.children.insert(i, child)'. This method also
sets the child's parent attribute appropriately.""" sets the child's parent attribute appropriately."""
child.parent = self child.parent = self
self.children.insert(i, child) self.children.insert(i, child)
self.changed()
def append_child(self, child): def append_child(self, child):
"""Equivalent to 'node.children.append(child)'. This method also """Equivalent to 'node.children.append(child)'. This method also
sets the child's parent attribute appropriately.""" sets the child's parent attribute appropriately."""
child.parent = self child.parent = self
self.children.append(child) self.children.append(child)
self.changed()
class Leaf(Base): class Leaf(Base):

View File

@ -123,8 +123,8 @@ class RefactoringTool(object):
logger=self.logger) logger=self.logger)
self.pre_order, self.post_order = self.get_fixers() self.pre_order, self.post_order = self.get_fixers()
self.pre_order_mapping = get_headnode_dict(self.pre_order) self.pre_order_heads = get_headnode_dict(self.pre_order)
self.post_order_mapping = get_headnode_dict(self.post_order) self.post_order_heads = get_headnode_dict(self.post_order)
self.files = [] # List of files that were or should be modified self.files = [] # List of files that were or should be modified
@ -294,8 +294,8 @@ class RefactoringTool(object):
for fixer in all_fixers: for fixer in all_fixers:
fixer.start_tree(tree, name) fixer.start_tree(tree, name)
self.traverse_by(self.pre_order_mapping, tree.pre_order()) self.traverse_by(self.pre_order_heads, tree.pre_order())
self.traverse_by(self.post_order_mapping, tree.post_order()) self.traverse_by(self.post_order_heads, tree.post_order())
for fixer in all_fixers: for fixer in all_fixers:
fixer.finish_tree(tree, name) fixer.finish_tree(tree, name)

View File

@ -1,58 +0,0 @@
#!/usr/bin/env python2.5
"""
This is a benchmarking script to test the speed of 2to3's pattern matching
system. It's equivalent to "refactor.py -f all" for every Python module
in sys.modules, but without engaging the actual transformations.
"""
__author__ = "Collin Winter <collinw at gmail.com>"
# Python imports
import os.path
import sys
from time import time
# Test imports
from .support import adjust_path
adjust_path()
# Local imports
from .. import refactor
### Mock code for refactor.py and the fixers
###############################################################################
class Options:
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
self.verbose = False
def dummy_transform(*args, **kwargs):
pass
### Collect list of modules to match against
###############################################################################
files = []
for mod in sys.modules.values():
if mod is None or not hasattr(mod, '__file__'):
continue
f = mod.__file__
if f.endswith('.pyc'):
f = f[:-1]
if f.endswith('.py'):
files.append(f)
### Set up refactor and run the benchmark
###############################################################################
options = Options(fix=["all"], print_function=False, doctests_only=False)
refactor = refactor.RefactoringTool(options)
for fixer in refactor.fixers:
# We don't want them to actually fix the tree, just match against it.
fixer.transform = dummy_transform
t = time()
for f in files:
print "Matching", f
refactor.refactor_file(f)
print "%d seconds to match %d files" % (time() - t, len(sys.modules))

View File

@ -293,30 +293,30 @@ class Test_intern(FixerTestCase):
def test_prefix_preservation(self): def test_prefix_preservation(self):
b = """x = intern( a )""" b = """x = intern( a )"""
a = """x = sys.intern( a )""" a = """import sys\nx = sys.intern( a )"""
self.check(b, a) self.check(b, a)
b = """y = intern("b" # test b = """y = intern("b" # test
)""" )"""
a = """y = sys.intern("b" # test a = """import sys\ny = sys.intern("b" # test
)""" )"""
self.check(b, a) self.check(b, a)
b = """z = intern(a+b+c.d, )""" b = """z = intern(a+b+c.d, )"""
a = """z = sys.intern(a+b+c.d, )""" a = """import sys\nz = sys.intern(a+b+c.d, )"""
self.check(b, a) self.check(b, a)
def test(self): def test(self):
b = """x = intern(a)""" b = """x = intern(a)"""
a = """x = sys.intern(a)""" a = """import sys\nx = sys.intern(a)"""
self.check(b, a) self.check(b, a)
b = """z = intern(a+b+c.d,)""" b = """z = intern(a+b+c.d,)"""
a = """z = sys.intern(a+b+c.d,)""" a = """import sys\nz = sys.intern(a+b+c.d,)"""
self.check(b, a) self.check(b, a)
b = """intern("y%s" % 5).replace("y", "")""" b = """intern("y%s" % 5).replace("y", "")"""
a = """sys.intern("y%s" % 5).replace("y", "")""" a = """import sys\nsys.intern("y%s" % 5).replace("y", "")"""
self.check(b, a) self.check(b, a)
# These should not be refactored # These should not be refactored
@ -337,6 +337,35 @@ class Test_intern(FixerTestCase):
s = """intern()""" s = """intern()"""
self.unchanged(s) self.unchanged(s)
class Test_reduce(FixerTestCase):
fixer = "reduce"
def test_simple_call(self):
b = "reduce(a, b, c)"
a = "from functools import reduce\nreduce(a, b, c)"
self.check(b, a)
def test_call_with_lambda(self):
b = "reduce(lambda x, y: x + y, seq)"
a = "from functools import reduce\nreduce(lambda x, y: x + y, seq)"
self.check(b, a)
def test_unchanged(self):
s = "reduce(a)"
self.unchanged(s)
s = "reduce(a, b=42)"
self.unchanged(s)
s = "reduce(a, b, c, d)"
self.unchanged(s)
s = "reduce(**c)"
self.unchanged(s)
s = "reduce()"
self.unchanged(s)
class Test_print(FixerTestCase): class Test_print(FixerTestCase):
fixer = "print" fixer = "print"
@ -1044,33 +1073,39 @@ class Test_long(FixerTestCase):
a = """z = type(x) in (int, int)""" a = """z = type(x) in (int, int)"""
self.check(b, a) self.check(b, a)
def test_4(self):
b = """a = 12L"""
a = """a = 12"""
self.check(b, a)
def test_5(self):
b = """b = 0x12l"""
a = """b = 0x12"""
self.check(b, a)
def test_unchanged_1(self):
s = """a = 12"""
self.unchanged(s)
def test_unchanged_2(self):
s = """b = 0x12"""
self.unchanged(s)
def test_unchanged_3(self):
s = """c = 3.14"""
self.unchanged(s)
def test_prefix_preservation(self): def test_prefix_preservation(self):
b = """x = long( x )""" b = """x = long( x )"""
a = """x = int( x )""" a = """x = int( x )"""
self.check(b, a) self.check(b, a)
class Test_isinstance(FixerTestCase):
fixer = "isinstance"
def test_remove_multiple_items(self):
b = """isinstance(x, (int, int, int))"""
a = """isinstance(x, int)"""
self.check(b, a)
b = """isinstance(x, (int, float, int, int, float))"""
a = """isinstance(x, (int, float))"""
self.check(b, a)
b = """isinstance(x, (int, float, int, int, float, str))"""
a = """isinstance(x, (int, float, str))"""
self.check(b, a)
b = """isinstance(foo() + bar(), (x(), y(), x(), int, int))"""
a = """isinstance(foo() + bar(), (x(), y(), x(), int))"""
self.check(b, a)
def test_prefix_preservation(self):
b = """if isinstance( foo(), ( bar, bar, baz )) : pass"""
a = """if isinstance( foo(), ( bar, baz )) : pass"""
self.check(b, a)
def test_unchanged(self):
self.unchanged("isinstance(x, (str, int))")
class Test_dict(FixerTestCase): class Test_dict(FixerTestCase):
fixer = "dict" fixer = "dict"
@ -1287,6 +1322,14 @@ class Test_xrange(FixerTestCase):
a = """x = list(range(10, 3, 9)) + [4]""" a = """x = list(range(10, 3, 9)) + [4]"""
self.check(b, a) self.check(b, a)
b = """x = range(10)[::-1]"""
a = """x = list(range(10))[::-1]"""
self.check(b, a)
b = """x = range(10) [3]"""
a = """x = list(range(10)) [3]"""
self.check(b, a)
def test_xrange_in_for(self): def test_xrange_in_for(self):
b = """for i in xrange(10):\n j=i""" b = """for i in xrange(10):\n j=i"""
a = """for i in range(10):\n j=i""" a = """for i in range(10):\n j=i"""
@ -1422,9 +1465,8 @@ class Test_xreadlines(FixerTestCase):
s = "foo(xreadlines)" s = "foo(xreadlines)"
self.unchanged(s) self.unchanged(s)
class Test_imports(FixerTestCase):
fixer = "imports" class ImportsFixerTests:
from ..fixes.fix_imports import MAPPING as modules
def test_import_module(self): def test_import_module(self):
for old, new in self.modules.items(): for old, new in self.modules.items():
@ -1522,18 +1564,36 @@ class Test_imports(FixerTestCase):
self.check(b, a) self.check(b, a)
class Test_imports(FixerTestCase, ImportsFixerTests):
fixer = "imports"
from ..fixes.fix_imports import MAPPING as modules
class Test_imports2(Test_imports): def test_multiple_imports(self):
b = """import urlparse, cStringIO"""
a = """import urllib.parse, io"""
self.check(b, a)
def test_multiple_imports_as(self):
b = """
import copy_reg as bar, HTMLParser as foo, urlparse
s = urlparse.spam(bar.foo())
"""
a = """
import copyreg as bar, html.parser as foo, urllib.parse
s = urllib.parse.spam(bar.foo())
"""
self.check(b, a)
class Test_imports2(FixerTestCase, ImportsFixerTests):
fixer = "imports2" fixer = "imports2"
from ..fixes.fix_imports2 import MAPPING as modules from ..fixes.fix_imports2 import MAPPING as modules
class Test_imports_fixer_order(Test_imports): class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests):
fixer = None
def setUp(self): def setUp(self):
Test_imports.setUp(self, ['imports', 'imports2']) super(Test_imports_fixer_order, self).setUp(['imports', 'imports2'])
from ..fixes.fix_imports2 import MAPPING as mapping2 from ..fixes.fix_imports2 import MAPPING as mapping2
self.modules = mapping2.copy() self.modules = mapping2.copy()
from ..fixes.fix_imports import MAPPING as mapping1 from ..fixes.fix_imports import MAPPING as mapping1

View File

@ -526,6 +526,33 @@ class Test_find_binding(support.TestCase):
b = 7""" b = 7"""
self.failIf(self.find_binding("a", s)) self.failIf(self.find_binding("a", s))
class Test_touch_import(support.TestCase):
def test_after_docstring(self):
node = parse('"""foo"""\nbar()')
fixer_util.touch_import(None, "foo", node)
self.assertEqual(str(node), '"""foo"""\nimport foo\nbar()\n\n')
def test_after_imports(self):
node = parse('"""foo"""\nimport bar\nbar()')
fixer_util.touch_import(None, "foo", node)
self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n')
def test_beginning(self):
node = parse('bar()')
fixer_util.touch_import(None, "foo", node)
self.assertEqual(str(node), 'import foo\nbar()\n\n')
def test_from_import(self):
node = parse('bar()')
fixer_util.touch_import("cgi", "escape", node)
self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n')
def test_name_import(self):
node = parse('bar()')
fixer_util.touch_import(None, "cgi", node)
self.assertEqual(str(node), 'import cgi\nbar()\n\n')
if __name__ == "__main__": if __name__ == "__main__":
import __main__ import __main__