Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Merged revisions 72491-72493 via svnmerge from 

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r72491 | benjamin.peterson | 2009-05-08 19:33:27 -0500 (Fri, 08 May 2009) | 7 lines

  make 2to3 use unicode internally on 2.x

  This started out as a fix for #2660, but became this large refactoring
  when I realized the dire state this was in. 2to3 now uses
  tokenize.detect_encoding to decode the files correctly into unicode.
........
  r72492 | benjamin.peterson | 2009-05-08 19:35:38 -0500 (Fri, 08 May 2009) | 1 line

  remove compat code
........
  r72493 | benjamin.peterson | 2009-05-08 19:54:15 -0500 (Fri, 08 May 2009) | 1 line

  add a test for \r\n newlines
........
This commit is contained in:
Benjamin Peterson 2009-05-09 01:01:14 +00:00
parent 5edb1a1b0a
commit 84ad84e0bb
60 changed files with 386 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Lib/lib2to3/fixer_base.py
View File

@ -94,14 +94,14 @@ class BaseFix(object):
"""
raise NotImplementedError()
def new_name(self, template="xxx_todo_changeme"):
def new_name(self, template=u"xxx_todo_changeme"):
"""Return a string suitable for use as an identifier
The new name is guaranteed not to conflict with other identifiers.
"""
name = template
while name in self.used_names:
name = template + str(self.numbers.next())
name = template + unicode(self.numbers.next())
self.used_names.add(name)
return name
@ -120,7 +120,7 @@ class BaseFix(object):
"""
lineno = node.get_lineno()
for_output = node.clone()
for_output.set_prefix("")
for_output.set_prefix(u"")
msg = "Line %d: could not convert: %s"
self.log_message(msg % (lineno, for_output))
if reason:

64
Lib/lib2to3/fixer_util.py
View File

@ -14,13 +14,13 @@ from . import patcomp
def KeywordArg(keyword, value):
return Node(syms.argument,
[keyword, Leaf(token.EQUAL, '='), value])
[keyword, Leaf(token.EQUAL, u'='), value])
def LParen():
return Leaf(token.LPAR, "(")
return Leaf(token.LPAR, u"(")
def RParen():
return Leaf(token.RPAR, ")")
return Leaf(token.RPAR, u")")
def Assign(target, source):
"""Build an assignment statement"""
@ -43,11 +43,11 @@ def Attr(obj, attr):
def Comma():
"""A comma leaf"""
return Leaf(token.COMMA, ",")
return Leaf(token.COMMA, u",")
def Dot():
"""A period (.) leaf"""
return Leaf(token.DOT, ".")
return Leaf(token.DOT, u".")
def ArgList(args, lparen=LParen(), rparen=RParen()):
"""A parenthesised argument list, used by Call()"""
@ -65,20 +65,20 @@ def Call(func_name, args=None, prefix=None):
def Newline():
"""A newline literal"""
return Leaf(token.NEWLINE, "\n")
return Leaf(token.NEWLINE, u"\n")
def BlankLine():
"""A blank line"""
return Leaf(token.NEWLINE, "")
return Leaf(token.NEWLINE, u"")
def Number(n, prefix=None):
return Leaf(token.NUMBER, n, prefix=prefix)
def Subscript(index_node):
"""A numeric or string subscript"""
return Node(syms.trailer, [Leaf(token.LBRACE, '['),
return Node(syms.trailer, [Leaf(token.LBRACE, u'['),
index_node,
Leaf(token.RBRACE, ']')])
Leaf(token.RBRACE, u']')])
def String(string, prefix=None):
"""A string leaf"""
@ -89,24 +89,24 @@ def ListComp(xp, fp, it, test=None):
If test is None, the "if test" part is omitted.
"""
xp.set_prefix("")
fp.set_prefix(" ")
it.set_prefix(" ")
for_leaf = Leaf(token.NAME, "for")
for_leaf.set_prefix(" ")
in_leaf = Leaf(token.NAME, "in")
in_leaf.set_prefix(" ")
xp.set_prefix(u"")
fp.set_prefix(u" ")
it.set_prefix(u" ")
for_leaf = Leaf(token.NAME, u"for")
for_leaf.set_prefix(u" ")
in_leaf = Leaf(token.NAME, u"in")
in_leaf.set_prefix(u" ")
inner_args = [for_leaf, fp, in_leaf, it]
if test:
test.set_prefix(" ")
if_leaf = Leaf(token.NAME, "if")
if_leaf.set_prefix(" ")
test.set_prefix(u" ")
if_leaf = Leaf(token.NAME, u"if")
if_leaf.set_prefix(u" ")
inner_args.append(Node(syms.comp_if, [if_leaf, test]))
inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)])
return Node(syms.atom,
[Leaf(token.LBRACE, "["),
[Leaf(token.LBRACE, u"["),
inner,
Leaf(token.RBRACE, "]")])
Leaf(token.RBRACE, u"]")])
def FromImport(package_name, name_leafs):
""" Return an import statement in the form:
@ -120,9 +120,9 @@ def FromImport(package_name, name_leafs):
# Pull the leaves out of their old tree
leaf.remove()
children = [Leaf(token.NAME, 'from'),
Leaf(token.NAME, package_name, prefix=" "),
Leaf(token.NAME, 'import', prefix=" "),
children = [Leaf(token.NAME, u'from'),
Leaf(token.NAME, package_name, prefix=u" "),
Leaf(token.NAME, u'import', prefix=u" "),
Node(syms.import_as_names, name_leafs)]
imp = Node(syms.import_from, children)
return imp
@ -141,8 +141,8 @@ def is_tuple(node):
and isinstance(node.children[0], Leaf)
and isinstance(node.children[1], Node)
and isinstance(node.children[2], Leaf)
and node.children[0].value == "("
and node.children[2].value == ")")
and node.children[0].value == u"("
and node.children[2].value == u")")
def is_list(node):
"""Does the node represent a list literal?"""
@ -150,8 +150,8 @@ def is_list(node):
and len(node.children) > 1
and isinstance(node.children[0], Leaf)
and isinstance(node.children[-1], Leaf)
and node.children[0].value == "["
and node.children[-1].value == "]")
and node.children[0].value == u"["
and node.children[-1].value == u"]")
###########################################################
@ -317,11 +317,11 @@ def touch_import(package, name, node):
if package is None:
import_ = Node(syms.import_name, [
Leaf(token.NAME, 'import'),
Leaf(token.NAME, name, prefix=' ')
Leaf(token.NAME, u'import'),
Leaf(token.NAME, name, prefix=u' ')
])
else:
import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')])
import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u' ')])
children = [import_, Newline()]
if add_newline_before:
@ -409,7 +409,7 @@ def _is_import_binding(node, name, package=None):
if package and unicode(node.children[1]).strip() != package:
return None
n = node.children[3]
if package and _find('as', n):
if package and _find(u'as', n):
# See test_from_import_as for explanation
return None
elif n.type == syms.import_as_names and _find(name, n):

6
Lib/lib2to3/fixes/fix_apply.py
View File

@ -46,12 +46,12 @@ class FixApply(fixer_base.BaseFix):
if kwds is not None:
kwds = kwds.clone()
kwds.set_prefix("")
l_newargs = [pytree.Leaf(token.STAR, "*"), args]
l_newargs = [pytree.Leaf(token.STAR, u"*"), args]
if kwds is not None:
l_newargs.extend([Comma(),
pytree.Leaf(token.DOUBLESTAR, "**"),
pytree.Leaf(token.DOUBLESTAR, u"**"),
kwds])
l_newargs[-2].set_prefix(" ") # that's the ** token
l_newargs[-2].set_prefix(u" ") # that's the ** token
# XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
# can be translated into f(x, y, *t) instead of f(*(x, y) + t)
#new = pytree.Node(syms.power, (func, ArgList(l_newargs)))

2
Lib/lib2to3/fixes/fix_basestring.py
View File

@ -10,4 +10,4 @@ class FixBasestring(fixer_base.BaseFix):
PATTERN = "'basestring'"
def transform(self, node, results):
return Name("str", prefix=node.get_prefix())
return Name(u"str", prefix=node.get_prefix())

2
Lib/lib2to3/fixes/fix_buffer.py
View File

@ -18,4 +18,4 @@ class FixBuffer(fixer_base.BaseFix):
def transform(self, node, results):
name = results["name"]
name.replace(Name("memoryview", prefix=name.get_prefix()))
name.replace(Name(u"memoryview", prefix=name.get_prefix()))

4
Lib/lib2to3/fixes/fix_callable.py
View File

@ -27,5 +27,5 @@ class FixCallable(fixer_base.BaseFix):
def transform(self, node, results):
func = results["func"]
args = [func.clone(), String(', '), String("'__call__'")]
return Call(Name("hasattr"), args, prefix=node.get_prefix())
args = [func.clone(), String(u', '), String(u"'__call__'")]
return Call(Name(u"hasattr"), args, prefix=node.get_prefix())

6
Lib/lib2to3/fixes/fix_dict.py
View File

@ -51,7 +51,7 @@ class FixDict(fixer_base.BaseFix):
tail = results["tail"]
syms = self.syms
method_name = method.value
isiter = method_name.startswith("iter")
isiter = method_name.startswith(u"iter")
if isiter:
method_name = method_name[4:]
assert method_name in ("keys", "items", "values"), repr(method)
@ -65,8 +65,8 @@ class FixDict(fixer_base.BaseFix):
results["parens"].clone()]
new = pytree.Node(syms.power, args)
if not special:
new.set_prefix("")
new = Call(Name(isiter and "iter" or "list"), [new])
new.set_prefix(u"")
new = Call(Name(isiter and u"iter" or u"list"), [new])
if tail:
new = pytree.Node(syms.power, [new] + tail)
new.set_prefix(node.get_prefix())

14
Lib/lib2to3/fixes/fix_except.py
View File

@ -30,7 +30,7 @@ from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms
def find_excepts(nodes):
for i, n in enumerate(nodes):
if n.type == syms.except_clause:
if n.children[0].value == 'except':
if n.children[0].value == u'except':
yield (n, nodes[i+2])
class FixExcept(fixer_base.BaseFix):
@ -52,13 +52,13 @@ class FixExcept(fixer_base.BaseFix):
for except_clause, e_suite in find_excepts(try_cleanup):
if len(except_clause.children) == 4:
(E, comma, N) = except_clause.children[1:4]
comma.replace(Name("as", prefix=" "))
comma.replace(Name(u"as", prefix=u" "))
if N.type != token.NAME:
# Generate a new N for the except clause
new_N = Name(self.new_name(), prefix=" ")
new_N = Name(self.new_name(), prefix=u" ")
target = N.clone()
target.set_prefix("")
target.set_prefix(u"")
N.replace(new_N)
new_N = new_N.clone()
@ -74,7 +74,7 @@ class FixExcept(fixer_base.BaseFix):
# The assignment is different if old_N is a tuple or list
# In that case, the assignment is old_N = new_N.args
if is_tuple(N) or is_list(N):
assign = Assign(target, Attr(new_N, Name('args')))
assign = Assign(target, Attr(new_N, Name(u'args')))
else:
assign = Assign(target, new_N)
@ -82,10 +82,10 @@ class FixExcept(fixer_base.BaseFix):
for child in reversed(suite_stmts[:i]):
e_suite.insert_child(0, child)
e_suite.insert_child(i, assign)
elif N.get_prefix() == "":
elif N.get_prefix() == u"":
# No space after a comma is legal; no space after "as",
# not so much.
N.set_prefix(" ")
N.set_prefix(u" ")
#TODO(cwinter) fix this when children becomes a smart list
children = [c.clone() for c in node.children[:3]] + try_cleanup + tail

2
Lib/lib2to3/fixes/fix_exec.py
View File

@ -36,4 +36,4 @@ class FixExec(fixer_base.BaseFix):
if c is not None:
args.extend([Comma(), c.clone()])
return Call(Name("exec"), args, prefix=node.get_prefix())
return Call(Name(u"exec"), args, prefix=node.get_prefix())

12
Lib/lib2to3/fixes/fix_execfile.py
View File

@ -31,21 +31,21 @@ class FixExecfile(fixer_base.BaseFix):
execfile_paren = node.children[-1].children[-1].clone()
# Construct open().read().
open_args = ArgList([filename.clone()], rparen=execfile_paren)
open_call = Node(syms.power, [Name("open"), open_args])
read = [Node(syms.trailer, [Dot(), Name('read')]),
open_call = Node(syms.power, [Name(u"open"), open_args])
read = [Node(syms.trailer, [Dot(), Name(u'read')]),
Node(syms.trailer, [LParen(), RParen()])]
open_expr = [open_call] + read
# Wrap the open call in a compile call. This is so the filename will be
# preserved in the execed code.
filename_arg = filename.clone()
filename_arg.set_prefix(" ")
exec_str = String("'exec'", " ")
filename_arg.set_prefix(u" ")
exec_str = String(u"'exec'", u" ")
compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str]
compile_call = Call(Name("compile"), compile_args, "")
compile_call = Call(Name(u"compile"), compile_args, u"")
# Finally, replace the execfile call with an exec call.
args = [compile_call]
if globals is not None:
args.extend([Comma(), globals.clone()])
if locals is not None:
args.extend([Comma(), locals.clone()])
return Call(Name("exec"), args, prefix=node.get_prefix())
return Call(Name(u"exec"), args, prefix=node.get_prefix())

10
Lib/lib2to3/fixes/fix_filter.py
View File

@ -60,16 +60,16 @@ class FixFilter(fixer_base.ConditionalFix):
results.get("xp").clone())
elif "none" in results:
new = ListComp(Name("_f"),
Name("_f"),
new = ListComp(Name(u"_f"),
Name(u"_f"),
results["seq"].clone(),
Name("_f"))
Name(u"_f"))
else:
if in_special_context(node):
return None
new = node.clone()
new.set_prefix("")
new = Call(Name("list"), [new])
new.set_prefix(u"")
new = Call(Name(u"list"), [new])
new.set_prefix(node.get_prefix())
return new

2
Lib/lib2to3/fixes/fix_funcattrs.py
View File

@ -15,5 +15,5 @@ class FixFuncattrs(fixer_base.BaseFix):
def transform(self, node, results):
attr = results["attr"][0]
attr.replace(Name(("__%s__" % attr.value[5:]),
attr.replace(Name((u"__%s__" % attr.value[5:]),
prefix=attr.get_prefix()))

2
Lib/lib2to3/fixes/fix_getcwdu.py
View File

@ -15,4 +15,4 @@ class FixGetcwdu(fixer_base.BaseFix):
def transform(self, node, results):
name = results["name"]
name.replace(Name("getcwd", prefix=name.get_prefix()))
name.replace(Name(u"getcwd", prefix=name.get_prefix()))

6
Lib/lib2to3/fixes/fix_has_key.py
View File

@ -91,10 +91,10 @@ class FixHasKey(fixer_base.BaseFix):
before = before[0]
else:
before = pytree.Node(syms.power, before)
before.set_prefix(" ")
n_op = Name("in", prefix=" ")
before.set_prefix(u" ")
n_op = Name(u"in", prefix=u" ")
if negation:
n_not = Name("not", prefix=" ")
n_not = Name(u"not", prefix=u" ")
n_op = pytree.Node(syms.comp_op, (n_not, n_op))
new = pytree.Node(syms.comparison, (arg, n_op, before))
if after:

12
Lib/lib2to3/fixes/fix_idioms.py
View File

@ -105,14 +105,14 @@ class FixIdioms(fixer_base.BaseFix):
T.set_prefix(" ")
test = Call(Name("isinstance"), [x, Comma(), T])
if "n" in results:
test.set_prefix(" ")
test = Node(syms.not_test, [Name("not"), test])
test.set_prefix(u" ")
test = Node(syms.not_test, [Name(u"not"), test])
test.set_prefix(node.get_prefix())
return test
def transform_while(self, node, results):
one = results["while"]
one.replace(Name("True", prefix=one.get_prefix()))
one.replace(Name(u"True", prefix=one.get_prefix()))
def transform_sort(self, node, results):
sort_stmt = results["sort"]
@ -121,11 +121,11 @@ class FixIdioms(fixer_base.BaseFix):
simple_expr = results.get("expr")
if list_call:
list_call.replace(Name("sorted", prefix=list_call.get_prefix()))
list_call.replace(Name(u"sorted", prefix=list_call.get_prefix()))
elif simple_expr:
new = simple_expr.clone()
new.set_prefix("")
simple_expr.replace(Call(Name("sorted"), [new],
new.set_prefix(u"")
simple_expr.replace(Call(Name(u"sorted"), [new],
prefix=simple_expr.get_prefix()))
else:
raise RuntimeError("should not have reached here")

2
Lib/lib2to3/fixes/fix_import.py
View File

@ -54,7 +54,7 @@ class FixImport(fixer_base.BaseFix):
while not hasattr(imp, 'value'):
imp = imp.children[0]
if self.probably_a_local_import(imp.value):
imp.value = "." + imp.value
imp.value = u"." + imp.value
imp.changed()
return node
else:

2
Lib/lib2to3/fixes/fix_imports.py
View File

@ -123,7 +123,7 @@ class FixImports(fixer_base.BaseFix):
import_mod = results.get("module_name")
if import_mod:
mod_name = import_mod.value
new_name = self.mapping[mod_name]
new_name = unicode(self.mapping[mod_name])
import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
if "name_import" in results:
# If it's not a "from x import x, y" or "import x as y" import,

4
Lib/lib2to3/fixes/fix_input.py
View File

@ -22,5 +22,5 @@ class FixInput(fixer_base.BaseFix):
return
new = node.clone()
new.set_prefix("")
return Call(Name("eval"), [new], prefix=node.get_prefix())
new.set_prefix(u"")
return Call(Name(u"eval"), [new], prefix=node.get_prefix())

4
Lib/lib2to3/fixes/fix_intern.py
View File

@ -34,11 +34,11 @@ class FixIntern(fixer_base.BaseFix):
if after:
after = [n.clone() for n in after]
new = pytree.Node(syms.power,
Attr(Name("sys"), Name("intern")) +
Attr(Name(u"sys"), Name(u"intern")) +
[pytree.Node(syms.trailer,
[results["lpar"].clone(),
newarglist,
results["rpar"].clone()])] + after)
new.set_prefix(node.get_prefix())
touch_import(None, 'sys', node)
touch_import(None, u'sys', node)
return new

2
Lib/lib2to3/fixes/fix_itertools.py
View File

@ -27,7 +27,7 @@ class FixItertools(fixer_base.BaseFix):
def transform(self, node, results):
prefix = None
func = results['func'][0]
if 'it' in results and func.value != 'ifilterfalse':
if 'it' in results and func.value != u'ifilterfalse':
dot, it = (results['dot'], results['it'])
# Remove the 'itertools'
prefix = it.get_prefix()

6
Lib/lib2to3/fixes/fix_itertools_imports.py
View File

@ -24,12 +24,12 @@ class FixItertoolsImports(fixer_base.BaseFix):
assert child.type == syms.import_as_name
name_node = child.children[0]
member_name = name_node.value
if member_name in ('imap', 'izip', 'ifilter'):
if member_name in (u'imap', u'izip', u'ifilter'):
child.value = None
child.remove()
elif member_name == 'ifilterfalse':
elif member_name == u'ifilterfalse':
node.changed()
name_node.value = 'filterfalse'
name_node.value = u'filterfalse'
# Make sure the import statement is still sane
children = imports.children[:] or [imports]

2
Lib/lib2to3/fixes/fix_long.py
View File

@ -13,7 +13,7 @@ class FixLong(fixer_base.BaseFix):
PATTERN = "'long'"
static_int = Name("int")
static_int = Name(u"int")
def transform(self, node, results):
if is_probably_builtin(node):

8
Lib/lib2to3/fixes/fix_map.py
View File

@ -63,8 +63,8 @@ class FixMap(fixer_base.ConditionalFix):
if node.parent.type == syms.simple_stmt:
self.warning(node, "You should use a for loop here")
new = node.clone()
new.set_prefix("")
new = Call(Name("list"), [new])
new.set_prefix(u"")
new = Call(Name(u"list"), [new])
elif "map_lambda" in results:
new = ListComp(results.get("xp").clone(),
results.get("fp").clone(),
@ -76,7 +76,7 @@ class FixMap(fixer_base.ConditionalFix):
if in_special_context(node):
return None
new = node.clone()
new.set_prefix("")
new = Call(Name("list"), [new])
new.set_prefix(u"")
new = Call(Name(u"list"), [new])
new.set_prefix(node.get_prefix())
return new

24
Lib/lib2to3/fixes/fix_metaclass.py
View File

@ -113,7 +113,7 @@ def find_metas(cls_node):
# Check if the expr_node is a simple assignment.
left_node = expr_node.children[0]
if isinstance(left_node, Leaf) and \
left_node.value == '__metaclass__':
left_node.value == u'__metaclass__':
# We found a assignment to __metaclass__.
fixup_simple_stmt(node, i, simple_node)
remove_trailing_newline(simple_node)
@ -182,9 +182,9 @@ class FixMetaclass(fixer_base.BaseFix):
# Node(classdef, ['class', 'name', ':', suite])
# 0 1 2 3
arglist = Node(syms.arglist, [])
node.insert_child(2, Leaf(token.RPAR, ')'))
node.insert_child(2, Leaf(token.RPAR, u')'))
node.insert_child(2, arglist)
node.insert_child(2, Leaf(token.LPAR, '('))
node.insert_child(2, Leaf(token.LPAR, u'('))
else:
raise ValueError("Unexpected class definition")
@ -194,16 +194,16 @@ class FixMetaclass(fixer_base.BaseFix):
orig_meta_prefix = meta_txt.get_prefix()
if arglist.children:
arglist.append_child(Leaf(token.COMMA, ','))
meta_txt.set_prefix(' ')
arglist.append_child(Leaf(token.COMMA, u','))
meta_txt.set_prefix(u' ')
else:
meta_txt.set_prefix('')
meta_txt.set_prefix(u'')
# compact the expression "metaclass = Meta" -> "metaclass=Meta"
expr_stmt = last_metaclass.children[0]
assert expr_stmt.type == syms.expr_stmt
expr_stmt.children[1].set_prefix('')
expr_stmt.children[2].set_prefix('')
expr_stmt.children[1].set_prefix(u'')
expr_stmt.children[2].set_prefix(u'')
arglist.append_child(last_metaclass)
@ -213,15 +213,15 @@ class FixMetaclass(fixer_base.BaseFix):
if not suite.children:
# one-liner that was just __metaclass_
suite.remove()
pass_leaf = Leaf(text_type, 'pass')
pass_leaf = Leaf(text_type, u'pass')
pass_leaf.set_prefix(orig_meta_prefix)
node.append_child(pass_leaf)
node.append_child(Leaf(token.NEWLINE, '\n'))
node.append_child(Leaf(token.NEWLINE, u'\n'))
elif len(suite.children) > 1 and \
(suite.children[-2].type == token.INDENT and
suite.children[-1].type == token.DEDENT):
# there was only one line in the class body and it was __metaclass__
pass_leaf = Leaf(text_type, 'pass')
pass_leaf = Leaf(text_type, u'pass')
suite.insert_child(-1, pass_leaf)
suite.insert_child(-1, Leaf(token.NEWLINE, '\n'))
suite.insert_child(-1, Leaf(token.NEWLINE, u'\n'))

2
Lib/lib2to3/fixes/fix_methodattrs.py
View File

@ -19,5 +19,5 @@ class FixMethodattrs(fixer_base.BaseFix):
def transform(self, node, results):
attr = results["attr"][0]
new = MAP[attr.value]
new = unicode(MAP[attr.value])
attr.replace(Name(new, prefix=attr.get_prefix()))

4
Lib/lib2to3/fixes/fix_ne.py
View File

@ -14,9 +14,9 @@ class FixNe(fixer_base.BaseFix):
def match(self, node):
# Override
return node.type == token.NOTEQUAL and node.value == "<>"
return node.type == token.NOTEQUAL and node.value == u"<>"
def transform(self, node, results):
new = pytree.Leaf(token.NOTEQUAL, "!=")
new = pytree.Leaf(token.NOTEQUAL, u"!=")
new.set_prefix(node.get_prefix())
return new

14
Lib/lib2to3/fixes/fix_next.py
View File

@ -35,7 +35,7 @@ class FixNext(fixer_base.BaseFix):
def start_tree(self, tree, filename):
super(FixNext, self).start_tree(tree, filename)
n = find_binding('next', tree)
n = find_binding(u'next', tree)
if n:
self.warning(n, bind_warning)
self.shadowed_next = True
@ -52,13 +52,13 @@ class FixNext(fixer_base.BaseFix):
if base:
if self.shadowed_next:
attr.replace(Name("__next__", prefix=attr.get_prefix()))
attr.replace(Name(u"__next__", prefix=attr.get_prefix()))
else:
base = [n.clone() for n in base]
base[0].set_prefix("")
node.replace(Call(Name("next", prefix=node.get_prefix()), base))
base[0].set_prefix(u"")
node.replace(Call(Name(u"next", prefix=node.get_prefix()), base))
elif name:
n = Name("__next__", prefix=name.get_prefix())
n = Name(u"__next__", prefix=name.get_prefix())
name.replace(n)
elif attr:
# We don't do this transformation if we're assigning to "x.next".
@ -66,10 +66,10 @@ class FixNext(fixer_base.BaseFix):
# so it's being done here.
if is_assign_target(node):
head = results["head"]
if "".join([str(n) for n in head]).strip() == '__builtin__':
if "".join([str(n) for n in head]).strip() == u'__builtin__':
self.warning(node, bind_warning)
return
attr.replace(Name("__next__"))
attr.replace(Name(u"__next__"))
elif "global" in results:
self.warning(node, bind_warning)
self.shadowed_next = True

2
Lib/lib2to3/fixes/fix_nonzero.py
View File

@ -16,5 +16,5 @@ class FixNonzero(fixer_base.BaseFix):
def transform(self, node, results):
name = results["name"]
new = Name("__bool__", prefix=name.get_prefix())
new = Name(u"__bool__", prefix=name.get_prefix())
name.replace(new)

8
Lib/lib2to3/fixes/fix_numliterals.py
View File

@ -15,13 +15,13 @@ class FixNumliterals(fixer_base.BaseFix):
def match(self, node):
# Override
return (node.type == token.NUMBER and
(node.value.startswith("0") or node.value[-1] in "Ll"))
(node.value.startswith(u"0") or node.value[-1] in u"Ll"))
def transform(self, node, results):
val = node.value
if val[-1] in 'Ll':
if val[-1] in u'Ll':
val = val[:-1]
elif val.startswith('0') and val.isdigit() and len(set(val)) > 1:
val = "0o" + val[1:]
elif val.startswith(u'0') and val.isdigit() and len(set(val)) > 1:
val = u"0o" + val[1:]
return Number(val, prefix=node.get_prefix())

2
Lib/lib2to3/fixes/fix_paren.py
View File

@ -37,6 +37,6 @@ class FixParen(fixer_base.BaseFix):
lparen = LParen()
lparen.set_prefix(target.get_prefix())
target.set_prefix("") # Make it hug the parentheses
target.set_prefix(u"") # Make it hug the parentheses
target.insert_child(0, lparen)
target.append_child(RParen())

22
Lib/lib2to3/fixes/fix_print.py
View File

@ -44,10 +44,10 @@ class FixPrint(fixer_base.ConditionalFix):
if bare_print:
# Special-case print all by itself
bare_print.replace(Call(Name("print"), [],
bare_print.replace(Call(Name(u"print"), [],
prefix=bare_print.get_prefix()))
return
assert node.children[0] == Name("print")
assert node.children[0] == Name(u"print")
args = node.children[1:]
if len(args) == 1 and parend_expr.match(args[0]):
# We don't want to keep sticking parens around an
@ -58,33 +58,33 @@ class FixPrint(fixer_base.ConditionalFix):
if args and args[-1] == Comma():
args = args[:-1]
end = " "
if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, ">>"):
if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, u">>"):
assert len(args) >= 2
file = args[1].clone()
args = args[3:] # Strip a possible comma after the file expression
# Now synthesize a print(args, sep=..., end=..., file=...) node.
l_args = [arg.clone() for arg in args]
if l_args:
l_args[0].set_prefix("")
l_args[0].set_prefix(u"")
if sep is not None or end is not None or file is not None:
if sep is not None:
self.add_kwarg(l_args, "sep", String(repr(sep)))
self.add_kwarg(l_args, u"sep", String(repr(sep)))
if end is not None:
self.add_kwarg(l_args, "end", String(repr(end)))
self.add_kwarg(l_args, u"end", String(repr(end)))
if file is not None:
self.add_kwarg(l_args, "file", file)
n_stmt = Call(Name("print"), l_args)
self.add_kwarg(l_args, u"file", file)
n_stmt = Call(Name(u"print"), l_args)
n_stmt.set_prefix(node.get_prefix())
return n_stmt
def add_kwarg(self, l_nodes, s_kwd, n_expr):
# XXX All this prefix-setting may lose comments (though rarely)
n_expr.set_prefix("")
n_expr.set_prefix(u"")
n_argument = pytree.Node(self.syms.argument,
(Name(s_kwd),
pytree.Leaf(token.EQUAL, "="),
pytree.Leaf(token.EQUAL, u"="),
n_expr))
if l_nodes:
l_nodes.append(Comma())
n_argument.set_prefix(" ")
n_argument.set_prefix(u" ")
l_nodes.append(n_argument)

12
Lib/lib2to3/fixes/fix_raise.py
View File

@ -56,7 +56,7 @@ class FixRaise(fixer_base.BaseFix):
if "val" not in results:
# One-argument raise
new = pytree.Node(syms.raise_stmt, [Name("raise"), exc])
new = pytree.Node(syms.raise_stmt, [Name(u"raise"), exc])
new.set_prefix(node.get_prefix())
return new
@ -64,19 +64,19 @@ class FixRaise(fixer_base.BaseFix):
if is_tuple(val):
args = [c.clone() for c in val.children[1:-1]]
else:
val.set_prefix("")
val.set_prefix(u"")
args = [val]
if "tb" in results:
tb = results["tb"].clone()
tb.set_prefix("")
tb.set_prefix(u"")
e = Call(exc, args)
with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])]
new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb)
with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])]
new = pytree.Node(syms.simple_stmt, [Name(u"raise")] + with_tb)
new.set_prefix(node.get_prefix())
return new
else:
return pytree.Node(syms.raise_stmt,
[Name("raise"), Call(exc, args)],
[Name(u"raise"), Call(exc, args)],
prefix=node.get_prefix())

2
Lib/lib2to3/fixes/fix_raw_input.py
View File

@ -13,4 +13,4 @@ class FixRawInput(fixer_base.BaseFix):
def transform(self, node, results):
name = results["name"]
name.replace(Name("input", prefix=name.get_prefix()))
name.replace(Name(u"input", prefix=name.get_prefix()))

2
Lib/lib2to3/fixes/fix_reduce.py
View File

@ -30,4 +30,4 @@ class FixReduce(fixer_base.BaseFix):
"""
def transform(self, node, results):
touch_import('functools', 'reduce', node)
touch_import(u'functools', u'reduce', node)

2
Lib/lib2to3/fixes/fix_renames.py
View File

@ -65,5 +65,5 @@ class FixRenames(fixer_base.BaseFix):
#import_mod = results.get("module")
if mod_name and attr_name:
new_attr = LOOKUP[(mod_name.value, attr_name.value)]
new_attr = unicode(LOOKUP[(mod_name.value, attr_name.value)])
attr_name.replace(Name(new_attr, prefix=attr_name.get_prefix()))

2
Lib/lib2to3/fixes/fix_repr.py
View File

@ -19,4 +19,4 @@ class FixRepr(fixer_base.BaseFix):
if expr.type == self.syms.testlist1:
expr = parenthesize(expr)
return Call(Name("repr"), [expr], prefix=node.get_prefix())
return Call(Name(u"repr"), [expr], prefix=node.get_prefix())

4
Lib/lib2to3/fixes/fix_set_literal.py
View File

@ -34,9 +34,9 @@ class FixSetLiteral(fixer_base.BaseFix):
items = results["items"]
# Build the contents of the literal
literal = [pytree.Leaf(token.LBRACE, "{")]
literal = [pytree.Leaf(token.LBRACE, u"{")]
literal.extend(n.clone() for n in items.children)
literal.append(pytree.Leaf(token.RBRACE, "}"))
literal.append(pytree.Leaf(token.RBRACE, u"}"))
# Set the prefix of the right brace to that of the ')' or ']'
literal[-1].set_prefix(items.next_sibling.get_prefix())
maker = pytree.Node(syms.dictsetmaker, literal)

2
Lib/lib2to3/fixes/fix_standarderror.py
View File

@ -15,4 +15,4 @@ class FixStandarderror(fixer_base.BaseFix):
"""
def transform(self, node, results):
return Name("Exception", prefix=node.get_prefix())
return Name(u"Exception", prefix=node.get_prefix())

4
Lib/lib2to3/fixes/fix_sys_exc.py
View File

@ -22,8 +22,8 @@ class FixSysExc(fixer_base.BaseFix):
sys_attr = results["attribute"][0]
index = Number(self.exc_info.index(sys_attr.value))
call = Call(Name("exc_info"), prefix=sys_attr.get_prefix())
attr = Attr(Name("sys"), call)
call = Call(Name(u"exc_info"), prefix=sys_attr.get_prefix())
attr = Attr(Name(u"sys"), call)
attr[1].children[0].set_prefix(results["dot"].get_prefix())
attr.append(Subscript(index))
return Node(syms.power, attr, prefix=node.get_prefix())

8
Lib/lib2to3/fixes/fix_throw.py
View File

@ -32,7 +32,7 @@ class FixThrow(fixer_base.BaseFix):
return
# Leave "g.throw(E)" alone
val = results.get("val")
val = results.get(u"val")
if val is None:
return
@ -40,17 +40,17 @@ class FixThrow(fixer_base.BaseFix):
if is_tuple(val):
args = [c.clone() for c in val.children[1:-1]]
else:
val.set_prefix("")
val.set_prefix(u"")
args = [val]
throw_args = results["args"]
if "tb" in results:
tb = results["tb"].clone()
tb.set_prefix("")
tb.set_prefix(u"")
e = Call(exc, args)
with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])]
with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])]
throw_args.replace(pytree.Node(syms.power, with_tb))
else:
throw_args.replace(Call(exc, args))

14
Lib/lib2to3/fixes/fix_tuple_params.py
View File

@ -55,7 +55,7 @@ class FixTupleParams(fixer_base.BaseFix):
else:
start = 0
indent = "; "
end = pytree.Leaf(token.INDENT, "")
end = pytree.Leaf(token.INDENT, u"")
# We need access to self for new_name(), and making this a method
# doesn't feel right. Closing over self and new_lines makes the
@ -63,10 +63,10 @@ class FixTupleParams(fixer_base.BaseFix):
def handle_tuple(tuple_arg, add_prefix=False):
n = Name(self.new_name())
arg = tuple_arg.clone()
arg.set_prefix("")
arg.set_prefix(u"")
stmt = Assign(arg, n.clone())
if add_prefix:
n.set_prefix(" ")
n.set_prefix(u" ")
tuple_arg.replace(n)
new_lines.append(pytree.Node(syms.simple_stmt,
[stmt, end.clone()]))
@ -91,7 +91,7 @@ class FixTupleParams(fixer_base.BaseFix):
# TODO(cwinter) suite-cleanup
after = start
if start == 0:
new_lines[0].set_prefix(" ")
new_lines[0].set_prefix(u" ")
elif is_docstring(suite[0].children[start]):
new_lines[0].set_prefix(indent)
after = start + 1
@ -109,7 +109,7 @@ class FixTupleParams(fixer_base.BaseFix):
# Replace lambda ((((x)))): x with lambda x: x
if inner.type == token.NAME:
inner = inner.clone()
inner.set_prefix(" ")
inner.set_prefix(u" ")
args.replace(inner)
return
@ -117,7 +117,7 @@ class FixTupleParams(fixer_base.BaseFix):
to_index = map_to_index(params)
tup_name = self.new_name(tuple_name(params))
new_param = Name(tup_name, prefix=" ")
new_param = Name(tup_name, prefix=u" ")
args.replace(new_param.clone())
for n in body.post_order():
if n.type == token.NAME and n.value in to_index:
@ -166,4 +166,4 @@ def tuple_name(param_list):
l.append(tuple_name(obj))
else:
l.append(obj)
return "_".join(l)
return u"_".join(l)

2
Lib/lib2to3/fixes/fix_types.py
View File

@ -56,7 +56,7 @@ class FixTypes(fixer_base.BaseFix):
PATTERN = '|'.join(_pats)
def transform(self, node, results):
new_value = _TYPE_MAPPING.get(results["name"].value)
new_value = unicode(_TYPE_MAPPING.get(results["name"].value))
if new_value:
return Name(new_value, prefix=node.get_prefix())
return None

10
Lib/lib2to3/fixes/fix_unicode.py
View File

@ -12,17 +12,17 @@ class FixUnicode(fixer_base.BaseFix):
def transform(self, node, results):
if node.type == token.NAME:
if node.value == "unicode":
if node.value == u"unicode":
new = node.clone()
new.value = "str"
new.value = u"str"
return new
if node.value == "unichr":
if node.value == u"unichr":
new = node.clone()
new.value = "chr"
new.value = u"chr"
return new
# XXX Warn when __unicode__ found?
elif node.type == token.STRING:
if re.match(r"[uU][rR]?[\'\"]", node.value):
if re.match(ur"[uU][rR]?[\'\"]", node.value):
new = node.clone()
new.value = new.value[1:]
return new

10
Lib/lib2to3/fixes/fix_ws_comma.py
View File

@ -17,8 +17,8 @@ class FixWsComma(fixer_base.BaseFix):
any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
"""
COMMA = pytree.Leaf(token.COMMA, ",")
COLON = pytree.Leaf(token.COLON, ":")
COMMA = pytree.Leaf(token.COMMA, u",")
COLON = pytree.Leaf(token.COLON, u":")
SEPS = (COMMA, COLON)
def transform(self, node, results):
@ -27,13 +27,13 @@ class FixWsComma(fixer_base.BaseFix):
for child in new.children:
if child in self.SEPS:
prefix = child.get_prefix()
if prefix.isspace() and "\n" not in prefix:
child.set_prefix("")
if prefix.isspace() and u"\n" not in prefix:
child.set_prefix(u"")
comma = True
else:
if comma:
prefix = child.get_prefix()
if not prefix:
child.set_prefix(" ")
child.set_prefix(u" ")
comma = False
return new

10
Lib/lib2to3/fixes/fix_xrange.py
View File

@ -19,22 +19,22 @@ class FixXrange(fixer_base.BaseFix):
def transform(self, node, results):
name = results["name"]
if name.value == "xrange":
if name.value == u"xrange":
return self.transform_xrange(node, results)
elif name.value == "range":
elif name.value == u"range":
return self.transform_range(node, results)
else:
raise ValueError(repr(name))
def transform_xrange(self, node, results):
name = results["name"]
name.replace(Name("range", prefix=name.get_prefix()))
name.replace(Name(u"range", prefix=name.get_prefix()))
def transform_range(self, node, results):
if not self.in_special_context(node):
range_call = Call(Name("range"), [results["args"].clone()])
range_call = Call(Name(u"range"), [results["args"].clone()])
# Encase the range call in list().
list_call = Call(Name("list"), [range_call],
list_call = Call(Name(u"list"), [range_call],
prefix=node.get_prefix())
# Put things that were after the range() call after the list call.
for n in results["rest"]:

2
Lib/lib2to3/fixes/fix_xreadlines.py
View File

@ -19,6 +19,6 @@ class FixXreadlines(fixer_base.BaseFix):
no_call = results.get("no_call")
if no_call:
no_call.replace(Name("__iter__", prefix=no_call.get_prefix()))
no_call.replace(Name(u"__iter__", prefix=no_call.get_prefix()))
else:
node.replace([x.clone() for x in results["call"]])

4
Lib/lib2to3/fixes/fix_zip.py
View File

@ -28,7 +28,7 @@ class FixZip(fixer_base.ConditionalFix):
return None
new = node.clone()
new.set_prefix("")
new = Call(Name("list"), [new])
new.set_prefix(u"")
new = Call(Name(u"list"), [new])
new.set_prefix(node.get_prefix())
return new

6
Lib/lib2to3/main.py
View File

@ -23,7 +23,7 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
self.errors.append((msg, args, kwargs))
self.logger.error(msg, *args, **kwargs)
def write_file(self, new_text, filename, old_text):
def write_file(self, new_text, filename, old_text, encoding):
if not self.nobackups:
# Make backup
backup = filename + ".bak"
@ -37,8 +37,8 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
except os.error, err:
self.log_message("Can't rename %s to %s", filename, backup)
# Actually write the new file
super(StdoutRefactoringTool, self).write_file(new_text,
filename, old_text)
write = super(StdoutRefactoringTool, self).write_file
write(new_text, filename, old_text, encoding)
if not self.nobackups:
shutil.copymode(backup, filename)

2
Lib/lib2to3/patcomp.py
View File

@ -133,7 +133,7 @@ class PatternCompiler(object):
assert len(nodes) >= 1
node = nodes[0]
if node.type == token.STRING:
value = literals.evalString(node.value)
value = unicode(literals.evalString(node.value))
return pytree.LeafPattern(content=value)
elif node.type == token.NAME:
value = node.value

7
Lib/lib2to3/pgen2/driver.py
View File

@ -16,6 +16,7 @@ __author__ = "Guido van Rossum <guido@python.org>"
__all__ = ["Driver", "load_grammar"]
# Python imports
import codecs
import os
import logging
import sys
@ -41,7 +42,7 @@ class Driver(object):
lineno = 1
column = 0
type = value = start = end = line_text = None
prefix = ""
prefix = u""
for quintuple in tokens:
type, value, start, end, line_text = quintuple
if start != (lineno, column):
@ -90,9 +91,9 @@ class Driver(object):
"""Parse a stream and return the syntax tree."""
return self.parse_stream_raw(stream, debug)
def parse_file(self, filename, debug=False):
def parse_file(self, filename, encoding=None, debug=False):
"""Parse a file and return the syntax tree."""
stream = open(filename)
stream = codecs.open(filename, "r", encoding)
try:
return self.parse_stream(stream, debug)
finally:

70
Lib/lib2to3/pgen2/tokenize.py
View File

@ -30,6 +30,7 @@ __credits__ = \
'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
import string, re
from codecs import BOM_UTF8, lookup
from lib2to3.pgen2.token import *
from . import token
@ -226,6 +227,75 @@ class Untokenizer:
startline = False
toks_append(tokval)
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
def detect_encoding(readline):
"""
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argment, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present,
but disagree, a SyntaxError will be raised. If the encoding cookie is an
invalid charset, raise a SyntaxError.
If no encoding is specified, then the default of 'utf-8' will be returned.
"""
bom_found = False
encoding = None
def read_or_stop():
try:
return readline()
except StopIteration:
return b''
def find_cookie(line):
try:
line_string = line.decode('ascii')
except UnicodeDecodeError:
return None
matches = cookie_re.findall(line_string)
if not matches:
return None
encoding = matches[0]
try:
codec = lookup(encoding)
except LookupError:
# This behaviour mimics the Python interpreter
raise SyntaxError("unknown encoding: " + encoding)
if bom_found and codec.name != 'utf-8':
# This behaviour mimics the Python interpreter
raise SyntaxError('encoding problem: utf-8')
return encoding
first = read_or_stop()
if first.startswith(BOM_UTF8):
bom_found = True
first = first[3:]
if not first:
return 'utf-8', []
encoding = find_cookie(first)
if encoding:
return encoding, [first]
second = read_or_stop()
if not second:
return 'utf-8', [first]
encoding = find_cookie(second)
if encoding:
return encoding, [first, second]
return 'utf-8', [first, second]
def untokenize(iterable):
"""Transform tokens back into Python source code.

20
Lib/lib2to3/pytree.py
View File

@ -213,9 +213,13 @@ class Base(object):
"""
next_sib = self.next_sibling
if next_sib is None:
return ""
return u""
return next_sib.get_prefix()
if sys.version_info < (3, 0):
def __str__(self):
return unicode(self).encode("ascii")
class Node(Base):
@ -245,13 +249,16 @@ class Node(Base):
type_repr(self.type),
self.children)
def __str__(self):
def __unicode__(self):
"""
Return a pretty string representation.
This reproduces the input source exactly.
"""
return "".join(map(str, self.children))
return u"".join(map(unicode, self.children))
if sys.version_info > (3, 0):
__str__ = __unicode__
def _eq(self, other):
"""Compare two nodes for equality."""
@ -353,13 +360,16 @@ class Leaf(Base):
self.type,
self.value)
def __str__(self):
def __unicode__(self):
"""
Return a pretty string representation.
This reproduces the input source exactly.
"""
return self.prefix + str(self.value)
return self.prefix + unicode(self.value)
if sys.version_info > (3, 0):
__str__ = __unicode__
def _eq(self, other):
"""Compare two nodes for equality."""

86
Lib/lib2to3/refactor.py
View File

@ -22,8 +22,7 @@ from collections import defaultdict
from itertools import chain
# Local imports
from .pgen2 import driver
from .pgen2 import tokenize
from .pgen2 import driver, tokenize
from . import pytree
from . import patcomp
@ -87,6 +86,25 @@ def get_fixers_from_package(pkg_name):
return [pkg_name + "." + fix_name
for fix_name in get_all_fix_names(pkg_name, False)]
def _identity(obj):
return obj
if sys.version_info < (3, 0):
import codecs
_open_with_encoding = codecs.open
# codecs.open doesn't translate newlines sadly.
def _from_system_newlines(input):
return input.replace(u"\r\n", u"\n")
def _to_system_newlines(input):
if os.linesep != "\n":
return input.replace(u"\n", os.linesep)
else:
return input
else:
_open_with_encoding = open
_from_system_newlines = _identity
_to_system_newlines = _identity
class FixerError(Exception):
"""A fixer could not be loaded."""
@ -213,29 +231,42 @@ class RefactoringTool(object):
# Modify dirnames in-place to remove subdirs with leading dots
dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")]
def refactor_file(self, filename, write=False, doctests_only=False):
"""Refactors a file."""
def _read_python_source(self, filename):
"""
Do our best to decode a Python source file correctly.
"""
try:
f = open(filename)
f = open(filename, "rb")
except IOError, err:
self.log_error("Can't open %s: %s", filename, err)
return
return None, None
try:
input = f.read() + "\n" # Silence certain parse errors
encoding = tokenize.detect_encoding(f.readline)[0]
finally:
f.close()
with _open_with_encoding(filename, "r", encoding=encoding) as f:
return _from_system_newlines(f.read()), encoding
def refactor_file(self, filename, write=False, doctests_only=False):
"""Refactors a file."""
input, encoding = self._read_python_source(filename)
if input is None:
# Reading the file failed.
return
input += u"\n" # Silence certain parse errors
if doctests_only:
self.log_debug("Refactoring doctests in %s", filename)
output = self.refactor_docstring(input, filename)
if output != input:
self.processed_file(output, filename, input, write=write)
self.processed_file(output, filename, input, write, encoding)
else:
self.log_debug("No doctest changes in %s", filename)
else:
tree = self.refactor_string(input, filename)
if tree and tree.was_changed:
# The [:-1] is to take off the \n we added earlier
self.processed_file(str(tree)[:-1], filename, write=write)
self.processed_file(unicode(tree)[:-1], filename,
write=write, encoding=encoding)
else:
self.log_debug("No changes in %s", filename)
@ -321,31 +352,26 @@ class RefactoringTool(object):
node.replace(new)
node = new
def processed_file(self, new_text, filename, old_text=None, write=False):
def processed_file(self, new_text, filename, old_text=None, write=False,
encoding=None):
"""
Called when a file has been refactored, and there are changes.
"""
self.files.append(filename)
if old_text is None:
try:
f = open(filename, "r")
except IOError, err:
self.log_error("Can't read %s: %s", filename, err)
old_text = self._read_python_source(filename)[0]
if old_text is None:
return
try:
old_text = f.read()
finally:
f.close()
if old_text == new_text:
self.log_debug("No changes to %s", filename)
return
self.print_output(diff_texts(old_text, new_text, filename))
if write:
self.write_file(new_text, filename, old_text)
self.write_file(new_text, filename, old_text, encoding)
else:
self.log_debug("Not writing changes to %s", filename)
def write_file(self, new_text, filename, old_text):
def write_file(self, new_text, filename, old_text, encoding=None):
"""Writes a string to a file.
It first shows a unified diff between the old text and the new text, and
@ -353,12 +379,12 @@ class RefactoringTool(object):
set.
"""
try:
f = open(filename, "w")
f = _open_with_encoding(filename, "w", encoding=encoding)
except os.error, err:
self.log_error("Can't create %s: %s", filename, err)
return
try:
f.write(new_text)
f.write(_to_system_newlines(new_text))
except os.error, err:
self.log_error("Can't write %s: %s", filename, err)
finally:
@ -398,7 +424,7 @@ class RefactoringTool(object):
indent = line[:i]
elif (indent is not None and
(line.startswith(indent + self.PS2) or
line == indent + self.PS2.rstrip() + "\n")):
line == indent + self.PS2.rstrip() + u"\n")):
block.append(line)
else:
if block is not None:
@ -410,7 +436,7 @@ class RefactoringTool(object):
if block is not None:
result.extend(self.refactor_doctest(block, block_lineno,
indent, filename))
return "".join(result)
return u"".join(result)
def refactor_doctest(self, block, lineno, indent, filename):
"""Refactors one doctest.
@ -425,7 +451,7 @@ class RefactoringTool(object):
except Exception, err:
if self.log.isEnabledFor(logging.DEBUG):
for line in block:
self.log_debug("Source: %s", line.rstrip("\n"))
self.log_debug("Source: %s", line.rstrip(u"\n"))
self.log_error("Can't parse docstring in %s line %s: %s: %s",
filename, lineno, err.__class__.__name__, err)
return block
@ -433,9 +459,9 @@ class RefactoringTool(object):
new = str(tree).splitlines(True)
# Undo the adjustment of the line numbers in wrap_toks() below.
clipped, new = new[:lineno-1], new[lineno-1:]
assert clipped == ["\n"] * (lineno-1), clipped
if not new[-1].endswith("\n"):
new[-1] += "\n"
assert clipped == [u"\n"] * (lineno-1), clipped
if not new[-1].endswith(u"\n"):
new[-1] += u"\n"
block = [indent + self.PS1 + new.pop(0)]
if new:
block += [indent + self.PS2 + line for line in new]
@ -497,8 +523,8 @@ class RefactoringTool(object):
for line in block:
if line.startswith(prefix):
yield line[len(prefix):]
elif line == prefix.rstrip() + "\n":
yield "\n"
elif line == prefix.rstrip() + u"\n":
yield u"\n"
else:
raise AssertionError("line=%r, prefix=%r" % (line, prefix))
prefix = prefix2

3
Lib/lib2to3/tests/data/crlf.py Normal file
View File

@ -0,0 +1,3 @@
print "hi"
print "Like bad Windows newlines?"

4
Lib/lib2to3/tests/data/different_encoding.py Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
print(u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')

15
Lib/lib2to3/tests/support.py
View File

@ -9,12 +9,9 @@ import os.path
import re
from textwrap import dedent
#sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
# Local imports
from .. import pytree
from .. import refactor
from ..pgen2 import driver
from lib2to3 import pytree, refactor
from lib2to3.pgen2 import driver
test_dir = os.path.dirname(__file__)
proj_dir = os.path.normpath(os.path.join(test_dir, ".."))
@ -25,19 +22,13 @@ driver = driver.Driver(grammar, convert=pytree.convert)
def parse_string(string):
return driver.parse_string(reformat(string), debug=True)
# Python 2.3's TestSuite is not iter()-able
if sys.version_info < (2, 4):
def TestSuite_iter(self):
return iter(self._tests)
unittest.TestSuite.__iter__ = TestSuite_iter
def run_all_tests(test_mod=None, tests=None):
if tests is None:
tests = unittest.TestLoader().loadTestsFromModule(test_mod)
unittest.TextTestRunner(verbosity=2).run(tests)
def reformat(string):
return dedent(string) + "\n\n"
return dedent(string) + u"\n\n"
def get_refactorer(fixers=None, options=None):
"""

2
Lib/lib2to3/tests/test_all_fixers.py
View File

@ -27,7 +27,7 @@ class Test_all(support.TestCase):
def test_all_project_files(self):
for filepath in support.all_project_files():
print "Fixing %s..." % filepath
self.refactor.refactor_string(open(filepath).read(), filepath)
self.refactor.refactor_file(filepath)
if __name__ == "__main__":

4
Lib/lib2to3/tests/test_fixers.py
View File

@ -25,7 +25,7 @@ class FixerTestCase(support.TestCase):
options = {"print_function" : False}
self.refactor = support.get_refactorer(fix_list, options)
self.fixer_log = []
self.filename = "<string>"
self.filename = u"<string>"
for fixer in chain(self.refactor.pre_order,
self.refactor.post_order):
@ -35,7 +35,7 @@ class FixerTestCase(support.TestCase):
before = support.reformat(before)
after = support.reformat(after)
tree = self.refactor.refactor_string(before, self.filename)
self.failUnlessEqual(after, str(tree))
self.failUnlessEqual(after, unicode(tree))
return tree
def check(self, before, after, ignore_warnings=False):

22
Lib/lib2to3/tests/test_parser.py
View File

@ -14,9 +14,9 @@ from .support import driver, test_dir
# Python imports
import os
import os.path
# Local imports
from lib2to3.pgen2 import tokenize
from ..pgen2.parse import ParseError
@ -150,13 +150,25 @@ class TestParserIdempotency(support.TestCase):
def test_all_project_files(self):
for filepath in support.all_project_files():
print "Parsing %s..." % filepath
tree = driver.parse_file(filepath, debug=True)
if diff(filepath, tree):
with open(filepath, "rb") as fp:
encoding = tokenize.detect_encoding(fp.readline)[0]
fp.seek(0)
source = fp.read()
if encoding:
source = source.decode(encoding)
tree = driver.parse_string(source)
new = unicode(tree)
if encoding:
new = new.encode(encoding)
if diff(filepath, new):
self.fail("Idempotency failed: %s" % filepath)
class TestLiterals(GrammarTest):
def validate(self, s):
driver.parse_string(support.dedent(s) + "\n\n")
def test_multiline_bytes_literals(self):
s = """
md5test(b"\xaa" * 80,
@ -185,10 +197,10 @@ class TestLiterals(GrammarTest):
self.validate(s)
def diff(fn, tree):
def diff(fn, result):
f = open("@", "w")
try:
f.write(str(tree))
f.write(result)
finally:
f.close()
try:

42
Lib/lib2to3/tests/test_refactor.py
View File

@ -14,7 +14,8 @@ from lib2to3 import refactor, pygram, fixer_base
from . import support
FIXER_DIR = os.path.join(os.path.dirname(__file__), "data/fixers")
TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers")
sys.path.append(FIXER_DIR)
try:
@ -22,6 +23,8 @@ try:
finally:
sys.path.pop()
_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes")
class TestRefactoringTool(unittest.TestCase):
def setUp(self):
@ -121,19 +124,40 @@ class TestRefactoringTool(unittest.TestCase):
+def cheese(): pass""".splitlines()
self.assertEqual(diff_lines[:-1], expected)
def test_refactor_file(self):
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
old_contents = open(test_file, "r").read()
rt = self.rt()
def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
def read_file():
with open(test_file, "rb") as fp:
return fp.read()
old_contents = read_file()
rt = self.rt(fixers=fixers)
rt.refactor_file(test_file)
self.assertEqual(old_contents, open(test_file, "r").read())
self.assertEqual(old_contents, read_file())
rt.refactor_file(test_file, True)
try:
self.assertNotEqual(old_contents, open(test_file, "r").read())
rt.refactor_file(test_file, True)
self.assertNotEqual(old_contents, read_file())
finally:
open(test_file, "w").write(old_contents)
with open(test_file, "wb") as fp:
fp.write(old_contents)
def test_refactor_file(self):
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
def test_file_encoding(self):
fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
self.check_file_refactoring(fn)
def test_crlf_newlines(self):
old_sep = os.linesep
os.linesep = "\r\n"
try:
fn = os.path.join(TEST_DATA_DIR, "crlf.py")
fixes = refactor.get_fixers_from_package("lib2to3.fixes")
self.check_file_refactoring(fn, fixes)
finally:
os.linesep = old_sep
def test_refactor_docstring(self):
rt = self.rt()