Merged revisions 72491-72493 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r72491 | benjamin.peterson | 2009-05-08 19:33:27 -0500 (Fri, 08 May 2009) | 7 lines

  make 2to3 use unicode internally on 2.x

  This started out as a fix for #2660, but became this large refactoring
  when I realized the dire state this was in. 2to3 now uses
  tokenize.detect_encoding to decode the files correctly into unicode.
........
  r72492 | benjamin.peterson | 2009-05-08 19:35:38 -0500 (Fri, 08 May 2009) | 1 line

  remove compat code
........
  r72493 | benjamin.peterson | 2009-05-08 19:54:15 -0500 (Fri, 08 May 2009) | 1 line

  add a test for \r\n newlines
........

Lib/lib2to3/fixer_base.py

@@ -94,14 +94,14 @@ class BaseFix(object):
         """
         raise NotImplementedError()
 
-    def new_name(self, template="xxx_todo_changeme"):
+    def new_name(self, template=u"xxx_todo_changeme"):
         """Return a string suitable for use as an identifier
 
         The new name is guaranteed not to conflict with other identifiers.
         """
         name = template
         while name in self.used_names:
-            name = template + str(self.numbers.next())
+            name = template + unicode(self.numbers.next())
         self.used_names.add(name)
         return name
 
@@ -120,7 +120,7 @@ class BaseFix(object):
         """
         lineno = node.get_lineno()
         for_output = node.clone()
-        for_output.set_prefix("")
+        for_output.set_prefix(u"")
         msg = "Line %d: could not convert: %s"
         self.log_message(msg % (lineno, for_output))
         if reason:

Lib/lib2to3/fixer_util.py

@@ -14,13 +14,13 @@ from . import patcomp
 
 def KeywordArg(keyword, value):
     return Node(syms.argument,
-                [keyword, Leaf(token.EQUAL, '='), value])
+                [keyword, Leaf(token.EQUAL, u'='), value])
 
 def LParen():
-    return Leaf(token.LPAR, "(")
+    return Leaf(token.LPAR, u"(")
 
 def RParen():
-    return Leaf(token.RPAR, ")")
+    return Leaf(token.RPAR, u")")
 
 def Assign(target, source):
     """Build an assignment statement"""
@@ -43,11 +43,11 @@ def Attr(obj, attr):
 
 def Comma():
     """A comma leaf"""
-    return Leaf(token.COMMA, ",")
+    return Leaf(token.COMMA, u",")
 
 def Dot():
     """A period (.) leaf"""
-    return Leaf(token.DOT, ".")
+    return Leaf(token.DOT, u".")
 
 def ArgList(args, lparen=LParen(), rparen=RParen()):
     """A parenthesised argument list, used by Call()"""
@@ -65,20 +65,20 @@ def Call(func_name, args=None, prefix=None):
 
 def Newline():
     """A newline literal"""
-    return Leaf(token.NEWLINE, "\n")
+    return Leaf(token.NEWLINE, u"\n")
 
 def BlankLine():
     """A blank line"""
-    return Leaf(token.NEWLINE, "")
+    return Leaf(token.NEWLINE, u"")
 
 def Number(n, prefix=None):
     return Leaf(token.NUMBER, n, prefix=prefix)
 
 def Subscript(index_node):
     """A numeric or string subscript"""
-    return Node(syms.trailer, [Leaf(token.LBRACE, '['),
+    return Node(syms.trailer, [Leaf(token.LBRACE, u'['),
                                index_node,
-                               Leaf(token.RBRACE, ']')])
+                               Leaf(token.RBRACE, u']')])
 
 def String(string, prefix=None):
     """A string leaf"""
@@ -89,24 +89,24 @@ def ListComp(xp, fp, it, test=None):
 
     If test is None, the "if test" part is omitted.
     """
-    xp.set_prefix("")
+    xp.set_prefix(u"")
-    fp.set_prefix(" ")
+    fp.set_prefix(u" ")
-    it.set_prefix(" ")
+    it.set_prefix(u" ")
-    for_leaf = Leaf(token.NAME, "for")
+    for_leaf = Leaf(token.NAME, u"for")
-    for_leaf.set_prefix(" ")
+    for_leaf.set_prefix(u" ")
-    in_leaf = Leaf(token.NAME, "in")
+    in_leaf = Leaf(token.NAME, u"in")
-    in_leaf.set_prefix(" ")
+    in_leaf.set_prefix(u" ")
     inner_args = [for_leaf, fp, in_leaf, it]
     if test:
-        test.set_prefix(" ")
+        test.set_prefix(u" ")
-        if_leaf = Leaf(token.NAME, "if")
+        if_leaf = Leaf(token.NAME, u"if")
-        if_leaf.set_prefix(" ")
+        if_leaf.set_prefix(u" ")
         inner_args.append(Node(syms.comp_if, [if_leaf, test]))
     inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)])
     return Node(syms.atom,
-                       [Leaf(token.LBRACE, "["),
+                       [Leaf(token.LBRACE, u"["),
                         inner,
-                        Leaf(token.RBRACE, "]")])
+                        Leaf(token.RBRACE, u"]")])
 
 def FromImport(package_name, name_leafs):
     """ Return an import statement in the form:
@@ -120,9 +120,9 @@ def FromImport(package_name, name_leafs):
         # Pull the leaves out of their old tree
         leaf.remove()
 
-    children = [Leaf(token.NAME, 'from'),
+    children = [Leaf(token.NAME, u'from'),
-                Leaf(token.NAME, package_name, prefix=" "),
+                Leaf(token.NAME, package_name, prefix=u" "),
-                Leaf(token.NAME, 'import', prefix=" "),
+                Leaf(token.NAME, u'import', prefix=u" "),
                 Node(syms.import_as_names, name_leafs)]
     imp = Node(syms.import_from, children)
     return imp
@@ -141,8 +141,8 @@ def is_tuple(node):
             and isinstance(node.children[0], Leaf)
             and isinstance(node.children[1], Node)
             and isinstance(node.children[2], Leaf)
-            and node.children[0].value == "("
+            and node.children[0].value == u"("
-            and node.children[2].value == ")")
+            and node.children[2].value == u")")
 
 def is_list(node):
     """Does the node represent a list literal?"""
@@ -150,8 +150,8 @@ def is_list(node):
             and len(node.children) > 1
             and isinstance(node.children[0], Leaf)
             and isinstance(node.children[-1], Leaf)
-            and node.children[0].value == "["
+            and node.children[0].value == u"["
-            and node.children[-1].value == "]")
+            and node.children[-1].value == u"]")
 
 
 ###########################################################
@@ -317,11 +317,11 @@ def touch_import(package, name, node):
 
     if package is None:
         import_ = Node(syms.import_name, [
-            Leaf(token.NAME, 'import'),
+            Leaf(token.NAME, u'import'),
-            Leaf(token.NAME, name, prefix=' ')
+            Leaf(token.NAME, name, prefix=u' ')
         ])
     else:
-        import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')])
+        import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u' ')])
 
     children = [import_, Newline()]
     if add_newline_before:
@@ -409,7 +409,7 @@ def _is_import_binding(node, name, package=None):
         if package and unicode(node.children[1]).strip() != package:
             return None
         n = node.children[3]
-        if package and _find('as', n):
+        if package and _find(u'as', n):
             # See test_from_import_as for explanation
             return None
         elif n.type == syms.import_as_names and _find(name, n):

Lib/lib2to3/fixes/fix_apply.py

@@ -46,12 +46,12 @@ class FixApply(fixer_base.BaseFix):
         if kwds is not None:
             kwds = kwds.clone()
             kwds.set_prefix("")
-        l_newargs = [pytree.Leaf(token.STAR, "*"), args]
+        l_newargs = [pytree.Leaf(token.STAR, u"*"), args]
         if kwds is not None:
             l_newargs.extend([Comma(),
-                              pytree.Leaf(token.DOUBLESTAR, "**"),
+                              pytree.Leaf(token.DOUBLESTAR, u"**"),
                               kwds])
-            l_newargs[-2].set_prefix(" ") # that's the ** token
+            l_newargs[-2].set_prefix(u" ") # that's the ** token
         # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
         # can be translated into f(x, y, *t) instead of f(*(x, y) + t)
         #new = pytree.Node(syms.power, (func, ArgList(l_newargs)))

Lib/lib2to3/fixes/fix_basestring.py

@@ -10,4 +10,4 @@ class FixBasestring(fixer_base.BaseFix):
     PATTERN = "'basestring'"
 
     def transform(self, node, results):
-        return Name("str", prefix=node.get_prefix())
+        return Name(u"str", prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_buffer.py

@@ -18,4 +18,4 @@ class FixBuffer(fixer_base.BaseFix):
 
     def transform(self, node, results):
         name = results["name"]
-        name.replace(Name("memoryview", prefix=name.get_prefix()))
+        name.replace(Name(u"memoryview", prefix=name.get_prefix()))

Lib/lib2to3/fixes/fix_callable.py

@@ -27,5 +27,5 @@ class FixCallable(fixer_base.BaseFix):
     def transform(self, node, results):
         func = results["func"]
 
-        args = [func.clone(), String(', '), String("'__call__'")]
+        args = [func.clone(), String(u', '), String(u"'__call__'")]
-        return Call(Name("hasattr"), args, prefix=node.get_prefix())
+        return Call(Name(u"hasattr"), args, prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_dict.py

@@ -51,7 +51,7 @@ class FixDict(fixer_base.BaseFix):
         tail = results["tail"]
         syms = self.syms
         method_name = method.value
-        isiter = method_name.startswith("iter")
+        isiter = method_name.startswith(u"iter")
         if isiter:
             method_name = method_name[4:]
         assert method_name in ("keys", "items", "values"), repr(method)
@@ -65,8 +65,8 @@ class FixDict(fixer_base.BaseFix):
                        results["parens"].clone()]
         new = pytree.Node(syms.power, args)
         if not special:
-            new.set_prefix("")
+            new.set_prefix(u"")
-            new = Call(Name(isiter and "iter" or "list"), [new])
+            new = Call(Name(isiter and u"iter" or u"list"), [new])
         if tail:
             new = pytree.Node(syms.power, [new] + tail)
         new.set_prefix(node.get_prefix())

Lib/lib2to3/fixes/fix_except.py

@@ -30,7 +30,7 @@ from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms
 def find_excepts(nodes):
     for i, n in enumerate(nodes):
         if n.type == syms.except_clause:
-            if n.children[0].value == 'except':
+            if n.children[0].value == u'except':
                 yield (n, nodes[i+2])
 
 class FixExcept(fixer_base.BaseFix):
@@ -52,13 +52,13 @@ class FixExcept(fixer_base.BaseFix):
         for except_clause, e_suite in find_excepts(try_cleanup):
             if len(except_clause.children) == 4:
                 (E, comma, N) = except_clause.children[1:4]
-                comma.replace(Name("as", prefix=" "))
+                comma.replace(Name(u"as", prefix=u" "))
 
                 if N.type != token.NAME:
                     # Generate a new N for the except clause
-                    new_N = Name(self.new_name(), prefix=" ")
+                    new_N = Name(self.new_name(), prefix=u" ")
                     target = N.clone()
-                    target.set_prefix("")
+                    target.set_prefix(u"")
                     N.replace(new_N)
                     new_N = new_N.clone()
 
@@ -74,7 +74,7 @@ class FixExcept(fixer_base.BaseFix):
                     # The assignment is different if old_N is a tuple or list
                     # In that case, the assignment is old_N = new_N.args
                     if is_tuple(N) or is_list(N):
-                        assign = Assign(target, Attr(new_N, Name('args')))
+                        assign = Assign(target, Attr(new_N, Name(u'args')))
                     else:
                         assign = Assign(target, new_N)
 
@@ -82,10 +82,10 @@ class FixExcept(fixer_base.BaseFix):
                     for child in reversed(suite_stmts[:i]):
                         e_suite.insert_child(0, child)
                     e_suite.insert_child(i, assign)
-                elif N.get_prefix() == "":
+                elif N.get_prefix() == u"":
                     # No space after a comma is legal; no space after "as",
                     # not so much.
-                    N.set_prefix(" ")
+                    N.set_prefix(u" ")
 
         #TODO(cwinter) fix this when children becomes a smart list
         children = [c.clone() for c in node.children[:3]] + try_cleanup + tail

Lib/lib2to3/fixes/fix_exec.py

@@ -36,4 +36,4 @@ class FixExec(fixer_base.BaseFix):
         if c is not None:
             args.extend([Comma(), c.clone()])
 
-        return Call(Name("exec"), args, prefix=node.get_prefix())
+        return Call(Name(u"exec"), args, prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_execfile.py

@@ -31,21 +31,21 @@ class FixExecfile(fixer_base.BaseFix):
         execfile_paren = node.children[-1].children[-1].clone()
         # Construct open().read().
         open_args = ArgList([filename.clone()], rparen=execfile_paren)
-        open_call = Node(syms.power, [Name("open"), open_args])
+        open_call = Node(syms.power, [Name(u"open"), open_args])
-        read = [Node(syms.trailer, [Dot(), Name('read')]),
+        read = [Node(syms.trailer, [Dot(), Name(u'read')]),
                 Node(syms.trailer, [LParen(), RParen()])]
         open_expr = [open_call] + read
         # Wrap the open call in a compile call. This is so the filename will be
         # preserved in the execed code.
         filename_arg = filename.clone()
-        filename_arg.set_prefix(" ")
+        filename_arg.set_prefix(u" ")
-        exec_str = String("'exec'", " ")
+        exec_str = String(u"'exec'", u" ")
         compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str]
-        compile_call = Call(Name("compile"), compile_args, "")
+        compile_call = Call(Name(u"compile"), compile_args, u"")
         # Finally, replace the execfile call with an exec call.
         args = [compile_call]
         if globals is not None:
             args.extend([Comma(), globals.clone()])
         if locals is not None:
             args.extend([Comma(), locals.clone()])
-        return Call(Name("exec"), args, prefix=node.get_prefix())
+        return Call(Name(u"exec"), args, prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_filter.py

@@ -60,16 +60,16 @@ class FixFilter(fixer_base.ConditionalFix):
                            results.get("xp").clone())
 
         elif "none" in results:
-            new = ListComp(Name("_f"),
+            new = ListComp(Name(u"_f"),
-                           Name("_f"),
+                           Name(u"_f"),
                            results["seq"].clone(),
-                           Name("_f"))
+                           Name(u"_f"))
 
         else:
             if in_special_context(node):
                 return None
             new = node.clone()
-            new.set_prefix("")
+            new.set_prefix(u"")
-            new = Call(Name("list"), [new])
+            new = Call(Name(u"list"), [new])
         new.set_prefix(node.get_prefix())
         return new

Lib/lib2to3/fixes/fix_funcattrs.py

@@ -15,5 +15,5 @@ class FixFuncattrs(fixer_base.BaseFix):
 
     def transform(self, node, results):
         attr = results["attr"][0]
-        attr.replace(Name(("__%s__" % attr.value[5:]),
+        attr.replace(Name((u"__%s__" % attr.value[5:]),
                           prefix=attr.get_prefix()))

Lib/lib2to3/fixes/fix_getcwdu.py

@@ -15,4 +15,4 @@ class FixGetcwdu(fixer_base.BaseFix):
 
     def transform(self, node, results):
         name = results["name"]
-        name.replace(Name("getcwd", prefix=name.get_prefix()))
+        name.replace(Name(u"getcwd", prefix=name.get_prefix()))

Lib/lib2to3/fixes/fix_has_key.py

@@ -91,10 +91,10 @@ class FixHasKey(fixer_base.BaseFix):
             before = before[0]
         else:
             before = pytree.Node(syms.power, before)
-        before.set_prefix(" ")
+        before.set_prefix(u" ")
-        n_op = Name("in", prefix=" ")
+        n_op = Name(u"in", prefix=u" ")
         if negation:
-            n_not = Name("not", prefix=" ")
+            n_not = Name(u"not", prefix=u" ")
             n_op = pytree.Node(syms.comp_op, (n_not, n_op))
         new = pytree.Node(syms.comparison, (arg, n_op, before))
         if after:

Lib/lib2to3/fixes/fix_idioms.py

@@ -105,14 +105,14 @@ class FixIdioms(fixer_base.BaseFix):
         T.set_prefix(" ")
         test = Call(Name("isinstance"), [x, Comma(), T])
         if "n" in results:
-            test.set_prefix(" ")
+            test.set_prefix(u" ")
-            test = Node(syms.not_test, [Name("not"), test])
+            test = Node(syms.not_test, [Name(u"not"), test])
         test.set_prefix(node.get_prefix())
         return test
 
     def transform_while(self, node, results):
         one = results["while"]
-        one.replace(Name("True", prefix=one.get_prefix()))
+        one.replace(Name(u"True", prefix=one.get_prefix()))
 
     def transform_sort(self, node, results):
         sort_stmt = results["sort"]
@@ -121,11 +121,11 @@ class FixIdioms(fixer_base.BaseFix):
         simple_expr = results.get("expr")
 
         if list_call:
-            list_call.replace(Name("sorted", prefix=list_call.get_prefix()))
+            list_call.replace(Name(u"sorted", prefix=list_call.get_prefix()))
         elif simple_expr:
             new = simple_expr.clone()
-            new.set_prefix("")
+            new.set_prefix(u"")
-            simple_expr.replace(Call(Name("sorted"), [new],
+            simple_expr.replace(Call(Name(u"sorted"), [new],
                                      prefix=simple_expr.get_prefix()))
         else:
             raise RuntimeError("should not have reached here")

Lib/lib2to3/fixes/fix_import.py

@@ -54,7 +54,7 @@ class FixImport(fixer_base.BaseFix):
             while not hasattr(imp, 'value'):
                 imp = imp.children[0]
             if self.probably_a_local_import(imp.value):
-                imp.value = "." + imp.value
+                imp.value = u"." + imp.value
                 imp.changed()
                 return node
         else:

Lib/lib2to3/fixes/fix_imports.py

@@ -123,7 +123,7 @@ class FixImports(fixer_base.BaseFix):
         import_mod = results.get("module_name")
         if import_mod:
             mod_name = import_mod.value
-            new_name = self.mapping[mod_name]
+            new_name = unicode(self.mapping[mod_name])
             import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
             if "name_import" in results:
                 # If it's not a "from x import x, y" or "import x as y" import,

Lib/lib2to3/fixes/fix_input.py

@@ -22,5 +22,5 @@ class FixInput(fixer_base.BaseFix):
             return
 
         new = node.clone()
-        new.set_prefix("")
+        new.set_prefix(u"")
-        return Call(Name("eval"), [new], prefix=node.get_prefix())
+        return Call(Name(u"eval"), [new], prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_intern.py

@@ -34,11 +34,11 @@ class FixIntern(fixer_base.BaseFix):
         if after:
             after = [n.clone() for n in after]
         new = pytree.Node(syms.power,
-                          Attr(Name("sys"), Name("intern")) +
+                          Attr(Name(u"sys"), Name(u"intern")) +
                           [pytree.Node(syms.trailer,
                                        [results["lpar"].clone(),
                                         newarglist,
                                         results["rpar"].clone()])] + after)
         new.set_prefix(node.get_prefix())
-        touch_import(None, 'sys', node)
+        touch_import(None, u'sys', node)
         return new

Lib/lib2to3/fixes/fix_itertools.py

@@ -27,7 +27,7 @@ class FixItertools(fixer_base.BaseFix):
     def transform(self, node, results):
         prefix = None
         func = results['func'][0]
-        if 'it' in results and func.value != 'ifilterfalse':
+        if 'it' in results and func.value != u'ifilterfalse':
             dot, it = (results['dot'], results['it'])
             # Remove the 'itertools'
             prefix = it.get_prefix()

Lib/lib2to3/fixes/fix_itertools_imports.py

@@ -24,12 +24,12 @@ class FixItertoolsImports(fixer_base.BaseFix):
                 assert child.type == syms.import_as_name
                 name_node = child.children[0]
             member_name = name_node.value
-            if member_name in ('imap', 'izip', 'ifilter'):
+            if member_name in (u'imap', u'izip', u'ifilter'):
                 child.value = None
                 child.remove()
-            elif member_name == 'ifilterfalse':
+            elif member_name == u'ifilterfalse':
                 node.changed()
-                name_node.value = 'filterfalse'
+                name_node.value = u'filterfalse'
 
         # Make sure the import statement is still sane
         children = imports.children[:] or [imports]

Lib/lib2to3/fixes/fix_long.py

@@ -13,7 +13,7 @@ class FixLong(fixer_base.BaseFix):
 
     PATTERN = "'long'"
 
-    static_int = Name("int")
+    static_int = Name(u"int")
 
     def transform(self, node, results):
         if is_probably_builtin(node):

Lib/lib2to3/fixes/fix_map.py

@@ -63,8 +63,8 @@ class FixMap(fixer_base.ConditionalFix):
         if node.parent.type == syms.simple_stmt:
             self.warning(node, "You should use a for loop here")
             new = node.clone()
-            new.set_prefix("")
+            new.set_prefix(u"")
-            new = Call(Name("list"), [new])
+            new = Call(Name(u"list"), [new])
         elif "map_lambda" in results:
             new = ListComp(results.get("xp").clone(),
                            results.get("fp").clone(),
@@ -76,7 +76,7 @@ class FixMap(fixer_base.ConditionalFix):
                 if in_special_context(node):
                     return None
                 new = node.clone()
-            new.set_prefix("")
+            new.set_prefix(u"")
-            new = Call(Name("list"), [new])
+            new = Call(Name(u"list"), [new])
         new.set_prefix(node.get_prefix())
         return new

Lib/lib2to3/fixes/fix_metaclass.py

@@ -113,7 +113,7 @@ def find_metas(cls_node):
                 # Check if the expr_node is a simple assignment.
                 left_node = expr_node.children[0]
                 if isinstance(left_node, Leaf) and \
-                        left_node.value == '__metaclass__':
+                        left_node.value == u'__metaclass__':
                     # We found a assignment to __metaclass__.
                     fixup_simple_stmt(node, i, simple_node)
                     remove_trailing_newline(simple_node)
@@ -182,9 +182,9 @@ class FixMetaclass(fixer_base.BaseFix):
             # Node(classdef, ['class', 'name', ':', suite])
             #                 0        1       2    3
             arglist = Node(syms.arglist, [])
-            node.insert_child(2, Leaf(token.RPAR, ')'))
+            node.insert_child(2, Leaf(token.RPAR, u')'))
             node.insert_child(2, arglist)
-            node.insert_child(2, Leaf(token.LPAR, '('))
+            node.insert_child(2, Leaf(token.LPAR, u'('))
         else:
             raise ValueError("Unexpected class definition")
 
@@ -194,16 +194,16 @@ class FixMetaclass(fixer_base.BaseFix):
         orig_meta_prefix = meta_txt.get_prefix()
 
         if arglist.children:
-            arglist.append_child(Leaf(token.COMMA, ','))
+            arglist.append_child(Leaf(token.COMMA, u','))
-            meta_txt.set_prefix(' ')
+            meta_txt.set_prefix(u' ')
         else:
-            meta_txt.set_prefix('')
+            meta_txt.set_prefix(u'')
 
         # compact the expression "metaclass = Meta" -> "metaclass=Meta"
         expr_stmt = last_metaclass.children[0]
         assert expr_stmt.type == syms.expr_stmt
-        expr_stmt.children[1].set_prefix('')
+        expr_stmt.children[1].set_prefix(u'')
-        expr_stmt.children[2].set_prefix('')
+        expr_stmt.children[2].set_prefix(u'')
 
         arglist.append_child(last_metaclass)
 
@@ -213,15 +213,15 @@ class FixMetaclass(fixer_base.BaseFix):
         if not suite.children:
             # one-liner that was just __metaclass_
             suite.remove()
-            pass_leaf = Leaf(text_type, 'pass')
+            pass_leaf = Leaf(text_type, u'pass')
             pass_leaf.set_prefix(orig_meta_prefix)
             node.append_child(pass_leaf)
-            node.append_child(Leaf(token.NEWLINE, '\n'))
+            node.append_child(Leaf(token.NEWLINE, u'\n'))
 
         elif len(suite.children) > 1 and \
                  (suite.children[-2].type == token.INDENT and
                   suite.children[-1].type == token.DEDENT):
             # there was only one line in the class body and it was __metaclass__
-            pass_leaf = Leaf(text_type, 'pass')
+            pass_leaf = Leaf(text_type, u'pass')
             suite.insert_child(-1, pass_leaf)
-            suite.insert_child(-1, Leaf(token.NEWLINE, '\n'))
+            suite.insert_child(-1, Leaf(token.NEWLINE, u'\n'))

Lib/lib2to3/fixes/fix_methodattrs.py

@@ -19,5 +19,5 @@ class FixMethodattrs(fixer_base.BaseFix):
 
     def transform(self, node, results):
         attr = results["attr"][0]
-        new = MAP[attr.value]
+        new = unicode(MAP[attr.value])
         attr.replace(Name(new, prefix=attr.get_prefix()))

Lib/lib2to3/fixes/fix_ne.py

@@ -14,9 +14,9 @@ class FixNe(fixer_base.BaseFix):
 
     def match(self, node):
         # Override
-        return node.type == token.NOTEQUAL and node.value == "<>"
+        return node.type == token.NOTEQUAL and node.value == u"<>"
 
     def transform(self, node, results):
-        new = pytree.Leaf(token.NOTEQUAL, "!=")
+        new = pytree.Leaf(token.NOTEQUAL, u"!=")
         new.set_prefix(node.get_prefix())
         return new

Lib/lib2to3/fixes/fix_next.py

@@ -35,7 +35,7 @@ class FixNext(fixer_base.BaseFix):
     def start_tree(self, tree, filename):
         super(FixNext, self).start_tree(tree, filename)
 
-        n = find_binding('next', tree)
+        n = find_binding(u'next', tree)
         if n:
             self.warning(n, bind_warning)
             self.shadowed_next = True
@@ -52,13 +52,13 @@ class FixNext(fixer_base.BaseFix):
 
         if base:
             if self.shadowed_next:
-                attr.replace(Name("__next__", prefix=attr.get_prefix()))
+                attr.replace(Name(u"__next__", prefix=attr.get_prefix()))
             else:
                 base = [n.clone() for n in base]
-                base[0].set_prefix("")
+                base[0].set_prefix(u"")
-                node.replace(Call(Name("next", prefix=node.get_prefix()), base))
+                node.replace(Call(Name(u"next", prefix=node.get_prefix()), base))
         elif name:
-            n = Name("__next__", prefix=name.get_prefix())
+            n = Name(u"__next__", prefix=name.get_prefix())
             name.replace(n)
         elif attr:
             # We don't do this transformation if we're assigning to "x.next".
@@ -66,10 +66,10 @@ class FixNext(fixer_base.BaseFix):
             #  so it's being done here.
             if is_assign_target(node):
                 head = results["head"]
-                if "".join([str(n) for n in head]).strip() == '__builtin__':
+                if "".join([str(n) for n in head]).strip() == u'__builtin__':
                     self.warning(node, bind_warning)
                 return
-            attr.replace(Name("__next__"))
+            attr.replace(Name(u"__next__"))
         elif "global" in results:
             self.warning(node, bind_warning)
             self.shadowed_next = True

Lib/lib2to3/fixes/fix_nonzero.py

@@ -16,5 +16,5 @@ class FixNonzero(fixer_base.BaseFix):
 
     def transform(self, node, results):
         name = results["name"]
-        new = Name("__bool__", prefix=name.get_prefix())
+        new = Name(u"__bool__", prefix=name.get_prefix())
         name.replace(new)

Lib/lib2to3/fixes/fix_numliterals.py

@@ -15,13 +15,13 @@ class FixNumliterals(fixer_base.BaseFix):
     def match(self, node):
         # Override
         return (node.type == token.NUMBER and
-                (node.value.startswith("0") or node.value[-1] in "Ll"))
+                (node.value.startswith(u"0") or node.value[-1] in u"Ll"))
 
     def transform(self, node, results):
         val = node.value
-        if val[-1] in 'Ll':
+        if val[-1] in u'Ll':
             val = val[:-1]
-        elif val.startswith('0') and val.isdigit() and len(set(val)) > 1:
+        elif val.startswith(u'0') and val.isdigit() and len(set(val)) > 1:
-            val = "0o" + val[1:]
+            val = u"0o" + val[1:]
 
         return Number(val, prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_paren.py

@@ -37,6 +37,6 @@ class FixParen(fixer_base.BaseFix):
 
         lparen = LParen()
         lparen.set_prefix(target.get_prefix())
-        target.set_prefix("") # Make it hug the parentheses
+        target.set_prefix(u"") # Make it hug the parentheses
         target.insert_child(0, lparen)
         target.append_child(RParen())

Lib/lib2to3/fixes/fix_print.py

@@ -44,10 +44,10 @@ class FixPrint(fixer_base.ConditionalFix):
 
         if bare_print:
             # Special-case print all by itself
-            bare_print.replace(Call(Name("print"), [],
+            bare_print.replace(Call(Name(u"print"), [],
                                prefix=bare_print.get_prefix()))
             return
-        assert node.children[0] == Name("print")
+        assert node.children[0] == Name(u"print")
         args = node.children[1:]
         if len(args) == 1 and parend_expr.match(args[0]):
             # We don't want to keep sticking parens around an
@@ -58,33 +58,33 @@ class FixPrint(fixer_base.ConditionalFix):
         if args and args[-1] == Comma():
             args = args[:-1]
             end = " "
-        if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, ">>"):
+        if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, u">>"):
             assert len(args) >= 2
             file = args[1].clone()
             args = args[3:] # Strip a possible comma after the file expression
         # Now synthesize a print(args, sep=..., end=..., file=...) node.
         l_args = [arg.clone() for arg in args]
         if l_args:
-            l_args[0].set_prefix("")
+            l_args[0].set_prefix(u"")
         if sep is not None or end is not None or file is not None:
             if sep is not None:
-                self.add_kwarg(l_args, "sep", String(repr(sep)))
+                self.add_kwarg(l_args, u"sep", String(repr(sep)))
             if end is not None:
-                self.add_kwarg(l_args, "end", String(repr(end)))
+                self.add_kwarg(l_args, u"end", String(repr(end)))
             if file is not None:
-                self.add_kwarg(l_args, "file", file)
+                self.add_kwarg(l_args, u"file", file)
-        n_stmt = Call(Name("print"), l_args)
+        n_stmt = Call(Name(u"print"), l_args)
         n_stmt.set_prefix(node.get_prefix())
         return n_stmt
 
     def add_kwarg(self, l_nodes, s_kwd, n_expr):
         # XXX All this prefix-setting may lose comments (though rarely)
-        n_expr.set_prefix("")
+        n_expr.set_prefix(u"")
         n_argument = pytree.Node(self.syms.argument,
                                  (Name(s_kwd),
-                                  pytree.Leaf(token.EQUAL, "="),
+                                  pytree.Leaf(token.EQUAL, u"="),
                                   n_expr))
         if l_nodes:
             l_nodes.append(Comma())
-            n_argument.set_prefix(" ")
+            n_argument.set_prefix(u" ")
         l_nodes.append(n_argument)

Lib/lib2to3/fixes/fix_raise.py

@@ -56,7 +56,7 @@ class FixRaise(fixer_base.BaseFix):
 
         if "val" not in results:
             # One-argument raise
-            new = pytree.Node(syms.raise_stmt, [Name("raise"), exc])
+            new = pytree.Node(syms.raise_stmt, [Name(u"raise"), exc])
             new.set_prefix(node.get_prefix())
             return new
 
@@ -64,19 +64,19 @@ class FixRaise(fixer_base.BaseFix):
         if is_tuple(val):
             args = [c.clone() for c in val.children[1:-1]]
         else:
-            val.set_prefix("")
+            val.set_prefix(u"")
             args = [val]
 
         if "tb" in results:
             tb = results["tb"].clone()
-            tb.set_prefix("")
+            tb.set_prefix(u"")
 
             e = Call(exc, args)
-            with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])]
+            with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])]
-            new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb)
+            new = pytree.Node(syms.simple_stmt, [Name(u"raise")] + with_tb)
             new.set_prefix(node.get_prefix())
             return new
         else:
             return pytree.Node(syms.raise_stmt,
-                               [Name("raise"), Call(exc, args)],
+                               [Name(u"raise"), Call(exc, args)],
                                prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_raw_input.py

@@ -13,4 +13,4 @@ class FixRawInput(fixer_base.BaseFix):
 
     def transform(self, node, results):
         name = results["name"]
-        name.replace(Name("input", prefix=name.get_prefix()))
+        name.replace(Name(u"input", prefix=name.get_prefix()))

Lib/lib2to3/fixes/fix_reduce.py

@@ -30,4 +30,4 @@ class FixReduce(fixer_base.BaseFix):
     """
 
     def transform(self, node, results):
-        touch_import('functools', 'reduce', node)
+        touch_import(u'functools', u'reduce', node)

Lib/lib2to3/fixes/fix_renames.py

@@ -65,5 +65,5 @@ class FixRenames(fixer_base.BaseFix):
         #import_mod = results.get("module")
 
         if mod_name and attr_name:
-            new_attr = LOOKUP[(mod_name.value, attr_name.value)]
+            new_attr = unicode(LOOKUP[(mod_name.value, attr_name.value)])
             attr_name.replace(Name(new_attr, prefix=attr_name.get_prefix()))

Lib/lib2to3/fixes/fix_repr.py

@@ -19,4 +19,4 @@ class FixRepr(fixer_base.BaseFix):
 
         if expr.type == self.syms.testlist1:
             expr = parenthesize(expr)
-        return Call(Name("repr"), [expr], prefix=node.get_prefix())
+        return Call(Name(u"repr"), [expr], prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_set_literal.py

@@ -34,9 +34,9 @@ class FixSetLiteral(fixer_base.BaseFix):
             items = results["items"]
 
         # Build the contents of the literal
-        literal = [pytree.Leaf(token.LBRACE, "{")]
+        literal = [pytree.Leaf(token.LBRACE, u"{")]
         literal.extend(n.clone() for n in items.children)
-        literal.append(pytree.Leaf(token.RBRACE, "}"))
+        literal.append(pytree.Leaf(token.RBRACE, u"}"))
         # Set the prefix of the right brace to that of the ')' or ']'
         literal[-1].set_prefix(items.next_sibling.get_prefix())
         maker = pytree.Node(syms.dictsetmaker, literal)

Lib/lib2to3/fixes/fix_standarderror.py

@@ -15,4 +15,4 @@ class FixStandarderror(fixer_base.BaseFix):
               """
 
     def transform(self, node, results):
-        return Name("Exception", prefix=node.get_prefix())
+        return Name(u"Exception", prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_sys_exc.py

@@ -22,8 +22,8 @@ class FixSysExc(fixer_base.BaseFix):
         sys_attr = results["attribute"][0]
         index = Number(self.exc_info.index(sys_attr.value))
 
-        call = Call(Name("exc_info"), prefix=sys_attr.get_prefix())
+        call = Call(Name(u"exc_info"), prefix=sys_attr.get_prefix())
-        attr = Attr(Name("sys"), call)
+        attr = Attr(Name(u"sys"), call)
         attr[1].children[0].set_prefix(results["dot"].get_prefix())
         attr.append(Subscript(index))
         return Node(syms.power, attr, prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_throw.py

@@ -32,7 +32,7 @@ class FixThrow(fixer_base.BaseFix):
             return
 
         # Leave "g.throw(E)" alone
-        val = results.get("val")
+        val = results.get(u"val")
         if val is None:
             return
 
@@ -40,17 +40,17 @@ class FixThrow(fixer_base.BaseFix):
         if is_tuple(val):
             args = [c.clone() for c in val.children[1:-1]]
         else:
-            val.set_prefix("")
+            val.set_prefix(u"")
             args = [val]
 
         throw_args = results["args"]
 
         if "tb" in results:
             tb = results["tb"].clone()
-            tb.set_prefix("")
+            tb.set_prefix(u"")
 
             e = Call(exc, args)
-            with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])]
+            with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])]
             throw_args.replace(pytree.Node(syms.power, with_tb))
         else:
             throw_args.replace(Call(exc, args))

Lib/lib2to3/fixes/fix_tuple_params.py

@@ -55,7 +55,7 @@ class FixTupleParams(fixer_base.BaseFix):
         else:
             start = 0
             indent = "; "
-            end = pytree.Leaf(token.INDENT, "")
+            end = pytree.Leaf(token.INDENT, u"")
 
         # We need access to self for new_name(), and making this a method
         #  doesn't feel right. Closing over self and new_lines makes the
@@ -63,10 +63,10 @@ class FixTupleParams(fixer_base.BaseFix):
         def handle_tuple(tuple_arg, add_prefix=False):
             n = Name(self.new_name())
             arg = tuple_arg.clone()
-            arg.set_prefix("")
+            arg.set_prefix(u"")
             stmt = Assign(arg, n.clone())
             if add_prefix:
-                n.set_prefix(" ")
+                n.set_prefix(u" ")
             tuple_arg.replace(n)
             new_lines.append(pytree.Node(syms.simple_stmt,
                                          [stmt, end.clone()]))
@@ -91,7 +91,7 @@ class FixTupleParams(fixer_base.BaseFix):
         # TODO(cwinter) suite-cleanup
         after = start
         if start == 0:
-            new_lines[0].set_prefix(" ")
+            new_lines[0].set_prefix(u" ")
         elif is_docstring(suite[0].children[start]):
             new_lines[0].set_prefix(indent)
             after = start + 1
@@ -109,7 +109,7 @@ class FixTupleParams(fixer_base.BaseFix):
         # Replace lambda ((((x)))): x  with lambda x: x
         if inner.type == token.NAME:
             inner = inner.clone()
-            inner.set_prefix(" ")
+            inner.set_prefix(u" ")
             args.replace(inner)
             return
 
@@ -117,7 +117,7 @@ class FixTupleParams(fixer_base.BaseFix):
         to_index = map_to_index(params)
         tup_name = self.new_name(tuple_name(params))
 
-        new_param = Name(tup_name, prefix=" ")
+        new_param = Name(tup_name, prefix=u" ")
         args.replace(new_param.clone())
         for n in body.post_order():
             if n.type == token.NAME and n.value in to_index:
@@ -166,4 +166,4 @@ def tuple_name(param_list):
             l.append(tuple_name(obj))
         else:
             l.append(obj)
-    return "_".join(l)
+    return u"_".join(l)

Lib/lib2to3/fixes/fix_types.py

@@ -56,7 +56,7 @@ class FixTypes(fixer_base.BaseFix):
     PATTERN = '|'.join(_pats)
 
     def transform(self, node, results):
-        new_value = _TYPE_MAPPING.get(results["name"].value)
+        new_value = unicode(_TYPE_MAPPING.get(results["name"].value))
         if new_value:
             return Name(new_value, prefix=node.get_prefix())
         return None

Lib/lib2to3/fixes/fix_unicode.py

@@ -12,17 +12,17 @@ class FixUnicode(fixer_base.BaseFix):
 
     def transform(self, node, results):
         if node.type == token.NAME:
-            if node.value == "unicode":
+            if node.value == u"unicode":
                 new = node.clone()
-                new.value = "str"
+                new.value = u"str"
                 return new
-            if node.value == "unichr":
+            if node.value == u"unichr":
                 new = node.clone()
-                new.value = "chr"
+                new.value = u"chr"
                 return new
             # XXX Warn when __unicode__ found?
         elif node.type == token.STRING:
-            if re.match(r"[uU][rR]?[\'\"]", node.value):
+            if re.match(ur"[uU][rR]?[\'\"]", node.value):
                 new = node.clone()
                 new.value = new.value[1:]
                 return new

Lib/lib2to3/fixes/fix_ws_comma.py

@@ -17,8 +17,8 @@ class FixWsComma(fixer_base.BaseFix):
     any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
     """
 
-    COMMA = pytree.Leaf(token.COMMA, ",")
+    COMMA = pytree.Leaf(token.COMMA, u",")
-    COLON = pytree.Leaf(token.COLON, ":")
+    COLON = pytree.Leaf(token.COLON, u":")
     SEPS = (COMMA, COLON)
 
     def transform(self, node, results):
@@ -27,13 +27,13 @@ class FixWsComma(fixer_base.BaseFix):
         for child in new.children:
             if child in self.SEPS:
                 prefix = child.get_prefix()
-                if prefix.isspace() and "\n" not in prefix:
+                if prefix.isspace() and u"\n" not in prefix:
-                    child.set_prefix("")
+                    child.set_prefix(u"")
                 comma = True
             else:
                 if comma:
                     prefix = child.get_prefix()
                     if not prefix:
-                        child.set_prefix(" ")
+                        child.set_prefix(u" ")
                 comma = False
         return new

Lib/lib2to3/fixes/fix_xrange.py

@@ -19,22 +19,22 @@ class FixXrange(fixer_base.BaseFix):
 
     def transform(self, node, results):
         name = results["name"]
-        if name.value == "xrange":
+        if name.value == u"xrange":
             return self.transform_xrange(node, results)
-        elif name.value == "range":
+        elif name.value == u"range":
             return self.transform_range(node, results)
         else:
             raise ValueError(repr(name))
 
     def transform_xrange(self, node, results):
         name = results["name"]
-        name.replace(Name("range", prefix=name.get_prefix()))
+        name.replace(Name(u"range", prefix=name.get_prefix()))
 
     def transform_range(self, node, results):
         if not self.in_special_context(node):
-            range_call = Call(Name("range"), [results["args"].clone()])
+            range_call = Call(Name(u"range"), [results["args"].clone()])
             # Encase the range call in list().
-            list_call = Call(Name("list"), [range_call],
+            list_call = Call(Name(u"list"), [range_call],
                              prefix=node.get_prefix())
             # Put things that were after the range() call after the list call.
             for n in results["rest"]:

Lib/lib2to3/fixes/fix_xreadlines.py

@@ -19,6 +19,6 @@ class FixXreadlines(fixer_base.BaseFix):
         no_call = results.get("no_call")
 
         if no_call:
-            no_call.replace(Name("__iter__", prefix=no_call.get_prefix()))
+            no_call.replace(Name(u"__iter__", prefix=no_call.get_prefix()))
         else:
             node.replace([x.clone() for x in results["call"]])

Lib/lib2to3/fixes/fix_zip.py

@@ -28,7 +28,7 @@ class FixZip(fixer_base.ConditionalFix):
             return None
 
         new = node.clone()
-        new.set_prefix("")
+        new.set_prefix(u"")
-        new = Call(Name("list"), [new])
+        new = Call(Name(u"list"), [new])
         new.set_prefix(node.get_prefix())
         return new

Lib/lib2to3/main.py

@@ -23,7 +23,7 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
         self.errors.append((msg, args, kwargs))
         self.logger.error(msg, *args, **kwargs)
 
-    def write_file(self, new_text, filename, old_text):
+    def write_file(self, new_text, filename, old_text, encoding):
         if not self.nobackups:
             # Make backup
             backup = filename + ".bak"
@@ -37,8 +37,8 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
             except os.error, err:
                 self.log_message("Can't rename %s to %s", filename, backup)
         # Actually write the new file
-        super(StdoutRefactoringTool, self).write_file(new_text,
+        write = super(StdoutRefactoringTool, self).write_file
-                                                      filename, old_text)
+        write(new_text, filename, old_text, encoding)
         if not self.nobackups:
             shutil.copymode(backup, filename)
 

Lib/lib2to3/patcomp.py

@@ -133,7 +133,7 @@ class PatternCompiler(object):
         assert len(nodes) >= 1
         node = nodes[0]
         if node.type == token.STRING:
-            value = literals.evalString(node.value)
+            value = unicode(literals.evalString(node.value))
             return pytree.LeafPattern(content=value)
         elif node.type == token.NAME:
             value = node.value

Lib/lib2to3/pgen2/driver.py

@@ -16,6 +16,7 @@ __author__ = "Guido van Rossum <guido@python.org>"
 __all__ = ["Driver", "load_grammar"]
 
 # Python imports
+import codecs
 import os
 import logging
 import sys
@@ -41,7 +42,7 @@ class Driver(object):
         lineno = 1
         column = 0
         type = value = start = end = line_text = None
-        prefix = ""
+        prefix = u""
         for quintuple in tokens:
             type, value, start, end, line_text = quintuple
             if start != (lineno, column):
@@ -90,9 +91,9 @@ class Driver(object):
         """Parse a stream and return the syntax tree."""
         return self.parse_stream_raw(stream, debug)
 
-    def parse_file(self, filename, debug=False):
+    def parse_file(self, filename, encoding=None, debug=False):
         """Parse a file and return the syntax tree."""
-        stream = open(filename)
+        stream = codecs.open(filename, "r", encoding)
         try:
             return self.parse_stream(stream, debug)
         finally:

Lib/lib2to3/pgen2/tokenize.py

@@ -30,6 +30,7 @@ __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 
 import string, re
+from codecs import BOM_UTF8, lookup
 from lib2to3.pgen2.token import *
 
 from . import token
@@ -226,6 +227,75 @@ class Untokenizer:
                 startline = False
             toks_append(tokval)
 
+cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+
+def detect_encoding(readline):
+    """
+    The detect_encoding() function is used to detect the encoding that should
+    be used to decode a Python source file. It requires one argment, readline,
+    in the same way as the tokenize() generator.
+
+    It will call readline a maximum of twice, and return the encoding used
+    (as a string) and a list of any lines (left as bytes) it has read
+    in.
+
+    It detects the encoding from the presence of a utf-8 bom or an encoding
+    cookie as specified in pep-0263. If both a bom and a cookie are present,
+    but disagree, a SyntaxError will be raised. If the encoding cookie is an
+    invalid charset, raise a SyntaxError.
+
+    If no encoding is specified, then the default of 'utf-8' will be returned.
+    """
+    bom_found = False
+    encoding = None
+    def read_or_stop():
+        try:
+            return readline()
+        except StopIteration:
+            return b''
+
+    def find_cookie(line):
+        try:
+            line_string = line.decode('ascii')
+        except UnicodeDecodeError:
+            return None
+
+        matches = cookie_re.findall(line_string)
+        if not matches:
+            return None
+        encoding = matches[0]
+        try:
+            codec = lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError("unknown encoding: " + encoding)
+
+        if bom_found and codec.name != 'utf-8':
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError('encoding problem: utf-8')
+        return encoding
+
+    first = read_or_stop()
+    if first.startswith(BOM_UTF8):
+        bom_found = True
+        first = first[3:]
+    if not first:
+        return 'utf-8', []
+
+    encoding = find_cookie(first)
+    if encoding:
+        return encoding, [first]
+
+    second = read_or_stop()
+    if not second:
+        return 'utf-8', [first]
+
+    encoding = find_cookie(second)
+    if encoding:
+        return encoding, [first, second]
+
+    return 'utf-8', [first, second]
+
 def untokenize(iterable):
     """Transform tokens back into Python source code.
 

Lib/lib2to3/pytree.py

@@ -213,9 +213,13 @@ class Base(object):
         """
         next_sib = self.next_sibling
         if next_sib is None:
-            return ""
+            return u""
         return next_sib.get_prefix()
 
+    if sys.version_info < (3, 0):
+        def __str__(self):
+            return unicode(self).encode("ascii")
+
 
 class Node(Base):
 
@@ -245,13 +249,16 @@ class Node(Base):
                                type_repr(self.type),
                                self.children)
 
-    def __str__(self):
+    def __unicode__(self):
         """
         Return a pretty string representation.
 
         This reproduces the input source exactly.
         """
-        return "".join(map(str, self.children))
+        return u"".join(map(unicode, self.children))
+
+    if sys.version_info > (3, 0):
+        __str__ = __unicode__
 
     def _eq(self, other):
         """Compare two nodes for equality."""
@@ -353,13 +360,16 @@ class Leaf(Base):
                                self.type,
                                self.value)
 
-    def __str__(self):
+    def __unicode__(self):
         """
         Return a pretty string representation.
 
         This reproduces the input source exactly.
         """
-        return self.prefix + str(self.value)
+        return self.prefix + unicode(self.value)
+
+    if sys.version_info > (3, 0):
+        __str__ = __unicode__
 
     def _eq(self, other):
         """Compare two nodes for equality."""

Lib/lib2to3/refactor.py

@@ -22,8 +22,7 @@ from collections import defaultdict
 from itertools import chain
 
 # Local imports
-from .pgen2 import driver
+from .pgen2 import driver, tokenize
-from .pgen2 import tokenize
 
 from . import pytree
 from . import patcomp
@@ -87,6 +86,25 @@ def get_fixers_from_package(pkg_name):
     return [pkg_name + "." + fix_name
             for fix_name in get_all_fix_names(pkg_name, False)]
 
+def _identity(obj):
+    return obj
+
+if sys.version_info < (3, 0):
+    import codecs
+    _open_with_encoding = codecs.open
+    # codecs.open doesn't translate newlines sadly.
+    def _from_system_newlines(input):
+        return input.replace(u"\r\n", u"\n")
+    def _to_system_newlines(input):
+        if os.linesep != "\n":
+            return input.replace(u"\n", os.linesep)
+        else:
+            return input
+else:
+    _open_with_encoding = open
+    _from_system_newlines = _identity
+    _to_system_newlines = _identity
+
 
 class FixerError(Exception):
     """A fixer could not be loaded."""
@@ -213,29 +231,42 @@ class RefactoringTool(object):
             # Modify dirnames in-place to remove subdirs with leading dots
             dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")]
 
-    def refactor_file(self, filename, write=False, doctests_only=False):
+    def _read_python_source(self, filename):
-        """Refactors a file."""
+        """
+        Do our best to decode a Python source file correctly.
+        """
         try:
-            f = open(filename)
+            f = open(filename, "rb")
         except IOError, err:
             self.log_error("Can't open %s: %s", filename, err)
-            return
+            return None, None
         try:
-            input = f.read() + "\n" # Silence certain parse errors
+            encoding = tokenize.detect_encoding(f.readline)[0]
         finally:
             f.close()
+        with _open_with_encoding(filename, "r", encoding=encoding) as f:
+            return _from_system_newlines(f.read()), encoding
+
+    def refactor_file(self, filename, write=False, doctests_only=False):
+        """Refactors a file."""
+        input, encoding = self._read_python_source(filename)
+        if input is None:
+            # Reading the file failed.
+            return
+        input += u"\n" # Silence certain parse errors
         if doctests_only:
             self.log_debug("Refactoring doctests in %s", filename)
             output = self.refactor_docstring(input, filename)
             if output != input:
-                self.processed_file(output, filename, input, write=write)
+                self.processed_file(output, filename, input, write, encoding)
             else:
                 self.log_debug("No doctest changes in %s", filename)
         else:
             tree = self.refactor_string(input, filename)
             if tree and tree.was_changed:
                 # The [:-1] is to take off the \n we added earlier
-                self.processed_file(str(tree)[:-1], filename, write=write)
+                self.processed_file(unicode(tree)[:-1], filename,
+                                    write=write, encoding=encoding)
             else:
                 self.log_debug("No changes in %s", filename)
 
@@ -321,31 +352,26 @@ class RefactoringTool(object):
                         node.replace(new)
                         node = new
 
-    def processed_file(self, new_text, filename, old_text=None, write=False):
+    def processed_file(self, new_text, filename, old_text=None, write=False,
+                       encoding=None):
         """
         Called when a file has been refactored, and there are changes.
         """
         self.files.append(filename)
         if old_text is None:
-            try:
+            old_text = self._read_python_source(filename)[0]
-                f = open(filename, "r")
+            if old_text is None:
-            except IOError, err:
-                self.log_error("Can't read %s: %s", filename, err)
                 return
-            try:
-                old_text = f.read()
-            finally:
-                f.close()
         if old_text == new_text:
             self.log_debug("No changes to %s", filename)
             return
         self.print_output(diff_texts(old_text, new_text, filename))
         if write:
-            self.write_file(new_text, filename, old_text)
+            self.write_file(new_text, filename, old_text, encoding)
         else:
             self.log_debug("Not writing changes to %s", filename)
 
-    def write_file(self, new_text, filename, old_text):
+    def write_file(self, new_text, filename, old_text, encoding=None):
         """Writes a string to a file.
 
         It first shows a unified diff between the old text and the new text, and
@@ -353,12 +379,12 @@ class RefactoringTool(object):
         set.
         """
         try:
-            f = open(filename, "w")
+            f = _open_with_encoding(filename, "w", encoding=encoding)
         except os.error, err:
             self.log_error("Can't create %s: %s", filename, err)
             return
         try:
-            f.write(new_text)
+            f.write(_to_system_newlines(new_text))
         except os.error, err:
             self.log_error("Can't write %s: %s", filename, err)
         finally:
@@ -398,7 +424,7 @@ class RefactoringTool(object):
                 indent = line[:i]
             elif (indent is not None and
                   (line.startswith(indent + self.PS2) or
-                   line == indent + self.PS2.rstrip() + "\n")):
+                   line == indent + self.PS2.rstrip() + u"\n")):
                 block.append(line)
             else:
                 if block is not None:
@@ -410,7 +436,7 @@ class RefactoringTool(object):
         if block is not None:
             result.extend(self.refactor_doctest(block, block_lineno,
                                                 indent, filename))
-        return "".join(result)
+        return u"".join(result)
 
     def refactor_doctest(self, block, lineno, indent, filename):
         """Refactors one doctest.
@@ -425,7 +451,7 @@ class RefactoringTool(object):
         except Exception, err:
             if self.log.isEnabledFor(logging.DEBUG):
                 for line in block:
-                    self.log_debug("Source: %s", line.rstrip("\n"))
+                    self.log_debug("Source: %s", line.rstrip(u"\n"))
             self.log_error("Can't parse docstring in %s line %s: %s: %s",
                            filename, lineno, err.__class__.__name__, err)
             return block
@@ -433,9 +459,9 @@ class RefactoringTool(object):
             new = str(tree).splitlines(True)
             # Undo the adjustment of the line numbers in wrap_toks() below.
             clipped, new = new[:lineno-1], new[lineno-1:]
-            assert clipped == ["\n"] * (lineno-1), clipped
+            assert clipped == [u"\n"] * (lineno-1), clipped
-            if not new[-1].endswith("\n"):
+            if not new[-1].endswith(u"\n"):
-                new[-1] += "\n"
+                new[-1] += u"\n"
             block = [indent + self.PS1 + new.pop(0)]
             if new:
                 block += [indent + self.PS2 + line for line in new]
@@ -497,8 +523,8 @@ class RefactoringTool(object):
         for line in block:
             if line.startswith(prefix):
                 yield line[len(prefix):]
-            elif line == prefix.rstrip() + "\n":
+            elif line == prefix.rstrip() + u"\n":
-                yield "\n"
+                yield u"\n"
             else:
                 raise AssertionError("line=%r, prefix=%r" % (line, prefix))
             prefix = prefix2

Lib/lib2to3/tests/data/crlf.py

@@ -0,0 +1,3 @@
+print "hi"
+
+print "Like bad Windows newlines?"

Lib/lib2to3/tests/data/different_encoding.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: iso-8859-1 -*-
+print(u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
+

Lib/lib2to3/tests/support.py

@@ -9,12 +9,9 @@ import os.path
 import re
 from textwrap import dedent
 
-#sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-
 # Local imports
-from .. import pytree
+from lib2to3 import pytree, refactor
-from .. import refactor
+from lib2to3.pgen2 import driver
-from ..pgen2 import driver
 
 test_dir = os.path.dirname(__file__)
 proj_dir = os.path.normpath(os.path.join(test_dir, ".."))
@@ -25,19 +22,13 @@ driver = driver.Driver(grammar, convert=pytree.convert)
 def parse_string(string):
     return driver.parse_string(reformat(string), debug=True)
 
-# Python 2.3's TestSuite is not iter()-able
-if sys.version_info < (2, 4):
-    def TestSuite_iter(self):
-        return iter(self._tests)
-    unittest.TestSuite.__iter__ = TestSuite_iter
-
 def run_all_tests(test_mod=None, tests=None):
     if tests is None:
         tests = unittest.TestLoader().loadTestsFromModule(test_mod)
     unittest.TextTestRunner(verbosity=2).run(tests)
 
 def reformat(string):
-    return dedent(string) + "\n\n"
+    return dedent(string) + u"\n\n"
 
 def get_refactorer(fixers=None, options=None):
     """

Lib/lib2to3/tests/test_all_fixers.py

@@ -27,7 +27,7 @@ class Test_all(support.TestCase):
     def test_all_project_files(self):
         for filepath in support.all_project_files():
             print "Fixing %s..." % filepath
-            self.refactor.refactor_string(open(filepath).read(), filepath)
+            self.refactor.refactor_file(filepath)
 
 
 if __name__ == "__main__":

Lib/lib2to3/tests/test_fixers.py

@@ -25,7 +25,7 @@ class FixerTestCase(support.TestCase):
         options = {"print_function" : False}
         self.refactor = support.get_refactorer(fix_list, options)
         self.fixer_log = []
-        self.filename = "<string>"
+        self.filename = u"<string>"
 
         for fixer in chain(self.refactor.pre_order,
                            self.refactor.post_order):
@@ -35,7 +35,7 @@ class FixerTestCase(support.TestCase):
         before = support.reformat(before)
         after = support.reformat(after)
         tree = self.refactor.refactor_string(before, self.filename)
-        self.failUnlessEqual(after, str(tree))
+        self.failUnlessEqual(after, unicode(tree))
         return tree
 
     def check(self, before, after, ignore_warnings=False):

Lib/lib2to3/tests/test_parser.py

@@ -14,9 +14,9 @@ from .support import driver, test_dir
 
 # Python imports
 import os
-import os.path
 
 # Local imports
+from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
 
 
@@ -150,13 +150,25 @@ class TestParserIdempotency(support.TestCase):
     def test_all_project_files(self):
         for filepath in support.all_project_files():
             print "Parsing %s..." % filepath
-            tree = driver.parse_file(filepath, debug=True)
+            with open(filepath, "rb") as fp:
-            if diff(filepath, tree):
+                encoding = tokenize.detect_encoding(fp.readline)[0]
+                fp.seek(0)
+                source = fp.read()
+                if encoding:
+                    source = source.decode(encoding)
+            tree = driver.parse_string(source)
+            new = unicode(tree)
+            if encoding:
+                new = new.encode(encoding)
+            if diff(filepath, new):
                 self.fail("Idempotency failed: %s" % filepath)
 
 
 class TestLiterals(GrammarTest):
 
+    def validate(self, s):
+        driver.parse_string(support.dedent(s) + "\n\n")
+
     def test_multiline_bytes_literals(self):
         s = """
             md5test(b"\xaa" * 80,
@@ -185,10 +197,10 @@ class TestLiterals(GrammarTest):
         self.validate(s)
 
 
-def diff(fn, tree):
+def diff(fn, result):
     f = open("@", "w")
     try:
-        f.write(str(tree))
+        f.write(result)
     finally:
         f.close()
     try:

Lib/lib2to3/tests/test_refactor.py

@@ -14,7 +14,8 @@ from lib2to3 import refactor, pygram, fixer_base
 from . import support
 
 
-FIXER_DIR = os.path.join(os.path.dirname(__file__), "data/fixers")
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers")
 
 sys.path.append(FIXER_DIR)
 try:
@@ -22,6 +23,8 @@ try:
 finally:
     sys.path.pop()
 
+_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes")
+
 class TestRefactoringTool(unittest.TestCase):
 
     def setUp(self):
@@ -121,19 +124,40 @@ class TestRefactoringTool(unittest.TestCase):
 +def cheese(): pass""".splitlines()
         self.assertEqual(diff_lines[:-1], expected)
 
-    def test_refactor_file(self):
+    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
-        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        def read_file():
-        old_contents = open(test_file, "r").read()
+            with open(test_file, "rb") as fp:
-        rt = self.rt()
+                return fp.read()
+        old_contents = read_file()
+        rt = self.rt(fixers=fixers)
 
         rt.refactor_file(test_file)
-        self.assertEqual(old_contents, open(test_file, "r").read())
+        self.assertEqual(old_contents, read_file())
 
-        rt.refactor_file(test_file, True)
         try:
-            self.assertNotEqual(old_contents, open(test_file, "r").read())
+            rt.refactor_file(test_file, True)
+            self.assertNotEqual(old_contents, read_file())
         finally:
-            open(test_file, "w").write(old_contents)
+            with open(test_file, "wb") as fp:
+                fp.write(old_contents)
+
+    def test_refactor_file(self):
+        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
+
+    def test_file_encoding(self):
+        fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
+        self.check_file_refactoring(fn)
+
+    def test_crlf_newlines(self):
+        old_sep = os.linesep
+        os.linesep = "\r\n"
+        try:
+            fn = os.path.join(TEST_DATA_DIR, "crlf.py")
+            fixes = refactor.get_fixers_from_package("lib2to3.fixes")
+            self.check_file_refactoring(fn, fixes)
+        finally:
+            os.linesep = old_sep
 
     def test_refactor_docstring(self):
         rt = self.rt()