From 42d26d94cc5fe633613c853c1e1b4b51f0855df4 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 Nov 2009 18:16:46 +0000 Subject: [PATCH] Merged revisions 76160-76161,76250,76252,76447,76506 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r76160 | benjamin.peterson | 2009-11-08 18:53:48 -0600 (Sun, 08 Nov 2009) | 1 line undeprecate the -p option; it's useful for converting python3 sources ........ r76161 | benjamin.peterson | 2009-11-08 19:05:37 -0600 (Sun, 08 Nov 2009) | 1 line simplify condition ........ r76250 | benjamin.peterson | 2009-11-13 16:56:48 -0600 (Fri, 13 Nov 2009) | 1 line fix handling of a utf-8 bom #7313 ........ r76252 | benjamin.peterson | 2009-11-13 16:58:36 -0600 (Fri, 13 Nov 2009) | 1 line remove pdb turd ........ r76447 | benjamin.peterson | 2009-11-22 18:17:40 -0600 (Sun, 22 Nov 2009) | 1 line #7375 fix nested transformations in fix_urllib ........ r76506 | benjamin.peterson | 2009-11-24 18:34:31 -0600 (Tue, 24 Nov 2009) | 1 line use generator expressions in any() ........ --- Lib/lib2to3/fixes/fix_imports.py | 2 +- Lib/lib2to3/fixes/fix_next.py | 2 +- Lib/lib2to3/fixes/fix_renames.py | 2 +- Lib/lib2to3/fixes/fix_urllib.py | 10 +++--- Lib/lib2to3/main.py | 11 +++---- Lib/lib2to3/pgen2/tokenize.py | 10 ++++-- Lib/lib2to3/refactor.py | 49 ++++++++++++++---------------- Lib/lib2to3/tests/data/bom.py | 3 ++ Lib/lib2to3/tests/test_fixers.py | 15 +++++++-- Lib/lib2to3/tests/test_refactor.py | 20 +++++++----- 10 files changed, 71 insertions(+), 53 deletions(-) create mode 100644 Lib/lib2to3/tests/data/bom.py diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py index 9c815e14dae..e1ad667f7bf 100644 --- a/Lib/lib2to3/fixes/fix_imports.py +++ b/Lib/lib2to3/fixes/fix_imports.py @@ -108,7 +108,7 @@ class FixImports(fixer_base.BaseFix): # Module usage could be in the trailer of an attribute lookup, so we # might have nested matches when "bare_with_attr" is present. if "bare_with_attr" not in results and \ - any([match(obj) for obj in attr_chain(node, "parent")]): + any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py index c70f72b0aa9..9514798f4ce 100644 --- a/Lib/lib2to3/fixes/fix_next.py +++ b/Lib/lib2to3/fixes/fix_next.py @@ -99,4 +99,4 @@ def find_assign(node): def is_subtree(root, node): if root == node: return True - return any([is_subtree(c, node) for c in root.children]) + return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py index ae398c7523d..a45345a9811 100644 --- a/Lib/lib2to3/fixes/fix_renames.py +++ b/Lib/lib2to3/fixes/fix_renames.py @@ -49,7 +49,7 @@ class FixRenames(fixer_base.BaseFix): match = super(FixRenames, self).match results = match(node) if results: - if any([match(obj) for obj in attr_chain(node, "parent")]): + if any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index d11220c5e18..db18ca84eef 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -63,7 +63,8 @@ def build_pattern(): yield """import_name< 'import' dotted_as_name< module_as=%r 'as' any > > """ % old_module - yield """power< module_dot=%r trailer< '.' member=%s > any* > + # bare_with_attr has a special significance for FixImports.match(). + yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > """ % (old_module, members) @@ -150,12 +151,11 @@ class FixUrllib(FixImports): def transform_dot(self, node, results): """Transform for calls to module members in code.""" - module_dot = results.get('module_dot') + module_dot = results.get('bare_with_attr') member = results.get('member') - # this may be a list of length one, or just a node + new_name = None if isinstance(member, list): member = member[0] - new_name = None for change in MAPPING[module_dot.value]: if member.value in change[1]: new_name = change[0] @@ -171,7 +171,7 @@ class FixUrllib(FixImports): self.transform_import(node, results) elif results.get('mod_member'): self.transform_member(node, results) - elif results.get('module_dot'): + elif results.get('bare_with_attr'): self.transform_dot(node, results) # Renaming and star imports are not supported for these modules. elif results.get('module_star'): diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py index 92388079b2c..736d5a6bd1c 100644 --- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -91,8 +91,7 @@ def main(fixer_pkg, args=None): parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations (fixes/fix_*.py)") parser.add_option("-p", "--print-function", action="store_true", - help="DEPRECATED Modify the grammar so that print() is " - "a function") + help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", @@ -104,12 +103,10 @@ def main(fixer_pkg, args=None): # Parse command line arguments refactor_stdin = False + flags = {} options, args = parser.parse_args(args) if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") - if options.print_function: - warn("-p is deprecated; " - "detection of from __future__ import print_function is automatic") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if options.list_fixes: @@ -127,6 +124,8 @@ def main(fixer_pkg, args=None): if options.write: print >> sys.stderr, "Can't write to stdin." return 2 + if options.print_function: + flags["print_function"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO @@ -147,7 +146,7 @@ def main(fixer_pkg, args=None): else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit), + rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index fb41a0af524..163c561100b 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -281,9 +281,13 @@ def detect_encoding(readline): # This behaviour mimics the Python interpreter raise SyntaxError("unknown encoding: " + encoding) - if bom_found and codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') + if bom_found: + if codec.name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + else: + # Allow it to be properly encoded and decoded. + encoding = 'utf-8-sig' return encoding first = read_or_stop() diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 97a540d7c65..df5456e6f5b 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -18,7 +18,6 @@ import logging import operator import collections import StringIO -import warnings from itertools import chain # Local imports @@ -139,26 +138,23 @@ def _detect_future_print(source): if have_docstring: break have_docstring = True - elif tp == token.NAME: - if value == u"from": - tp, value = advance() - if tp != token.NAME and value != u"__future__": - break - tp, value = advance() - if tp != token.NAME and value != u"import": - break - tp, value = advance() - if tp == token.OP and value == u"(": - tp, value = advance() - while tp == token.NAME: - if value == u"print_function": - return True - tp, value = advance() - if tp != token.OP and value != u",": - break - tp, value = advance() - else: + elif tp == token.NAME and value == u"from": + tp, value = advance() + if tp != token.NAME and value != u"__future__": break + tp, value = advance() + if tp != token.NAME and value != u"import": + break + tp, value = advance() + if tp == token.OP and value == u"(": + tp, value = advance() + while tp == token.NAME: + if value == u"print_function": + return True + tp, value = advance() + if tp != token.OP and value != u",": + break + tp, value = advance() else: break except StopIteration: @@ -172,7 +168,7 @@ class FixerError(Exception): class RefactoringTool(object): - _default_options = {} + _default_options = {"print_function" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -189,15 +185,16 @@ class RefactoringTool(object): self.explicit = explicit or [] self.options = self._default_options.copy() if options is not None: - if "print_function" in options: - warnings.warn("the 'print_function' option is deprecated", - DeprecationWarning) self.options.update(options) + if self.options["print_function"]: + self.grammar = pygram.python_grammar_no_print_statement + else: + self.grammar = pygram.python_grammar self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] self.wrote = False - self.driver = driver.Driver(pygram.python_grammar, + self.driver = driver.Driver(self.grammar, convert=pytree.convert, logger=self.logger) self.pre_order, self.post_order = self.get_fixers() @@ -353,7 +350,7 @@ class RefactoringTool(object): name, err.__class__.__name__, err) return finally: - self.driver.grammar = pygram.python_grammar + self.driver.grammar = self.grammar self.log_debug("Refactoring %s", name) self.refactor_tree(tree, name) return tree diff --git a/Lib/lib2to3/tests/data/bom.py b/Lib/lib2to3/tests/data/bom.py new file mode 100644 index 00000000000..ecb782a30d4 --- /dev/null +++ b/Lib/lib2to3/tests/data/bom.py @@ -0,0 +1,3 @@ +# coding: utf-8 +print "BOM BOOM!" + diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index cad2a7d1de9..eb0b4b420ba 100755 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -1753,6 +1753,8 @@ class Test_urllib(FixerTestCase): for old, changes in self.modules.items(): for new, members in changes: for member in members: + new_import = ", ".join([n for (n, mems) + in self.modules[old]]) b = """ import %s foo(%s.%s) @@ -1760,9 +1762,16 @@ class Test_urllib(FixerTestCase): a = """ import %s foo(%s.%s) - """ % (", ".join([n for (n, mems) - in self.modules[old]]), - new, member) + """ % (new_import, new, member) + self.check(b, a) + b = """ + import %s + %s.%s(%s.%s) + """ % (old, old, member, old, member) + a = """ + import %s + %s.%s(%s.%s) + """ % (new_import, new, member, new, member) self.check(b, a) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index cc47b9d9c24..1729027d90a 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -4,6 +4,7 @@ Unit tests for refactor.py. import sys import os +import codecs import operator import StringIO import tempfile @@ -45,12 +46,10 @@ class TestRefactoringTool(unittest.TestCase): return refactor.RefactoringTool(fixers, options, explicit) def test_print_function_option(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True}) - self.assertEqual(len(w), 1) - msg, = w - self.assertTrue(msg.category is DeprecationWarning) + rt = self.rt({"print_function" : True}) + self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement) + self.assertTrue(rt.driver.grammar is + pygram.python_grammar_no_print_statement) def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] @@ -179,10 +178,12 @@ from __future__ import print_function""" try: rt.refactor_file(test_file, True) - self.assertNotEqual(old_contents, read_file()) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) finally: with open(test_file, "wb") as fp: fp.write(old_contents) + return new_contents def test_refactor_file(self): test_file = os.path.join(FIXER_DIR, "parrot_example.py") @@ -223,6 +224,11 @@ from __future__ import print_function""" fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_bom(self): + fn = os.path.join(TEST_DATA_DIR, "bom.py") + data = self.check_file_refactoring(fn) + self.assertTrue(data.startswith(codecs.BOM_UTF8)) + def test_crlf_newlines(self): old_sep = os.linesep os.linesep = "\r\n"