From 39778f6b6f5ce0f162ed7feaebdf370b6ea4daf8 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 Nov 2009 18:37:12 +0000 Subject: [PATCH] Merged revisions 76518 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/py3k ................ r76518 | benjamin.peterson | 2009-11-25 12:34:42 -0600 (Wed, 25 Nov 2009) | 60 lines Merged revisions 76259,76326,76376-76377,76430,76471,76517 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ................ r76259 | georg.brandl | 2009-11-14 05:50:51 -0600 (Sat, 14 Nov 2009) | 1 line Fix terminology. ................ r76326 | georg.brandl | 2009-11-16 10:44:05 -0600 (Mon, 16 Nov 2009) | 1 line #7302: fix link. ................ r76376 | georg.brandl | 2009-11-18 13:39:14 -0600 (Wed, 18 Nov 2009) | 1 line upcase Python ................ r76377 | georg.brandl | 2009-11-18 14:05:15 -0600 (Wed, 18 Nov 2009) | 1 line Fix markup. ................ r76430 | r.david.murray | 2009-11-20 07:29:43 -0600 (Fri, 20 Nov 2009) | 2 lines Issue 7363: fix indentation in socketserver udpserver example. ................ r76471 | georg.brandl | 2009-11-23 13:53:19 -0600 (Mon, 23 Nov 2009) | 1 line #7345: fix arguments of formatyear(). ................ r76517 | benjamin.peterson | 2009-11-25 12:16:46 -0600 (Wed, 25 Nov 2009) | 29 lines Merged revisions 76160-76161,76250,76252,76447,76506 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r76160 | benjamin.peterson | 2009-11-08 18:53:48 -0600 (Sun, 08 Nov 2009) | 1 line undeprecate the -p option; it's useful for converting python3 sources ........ r76161 | benjamin.peterson | 2009-11-08 19:05:37 -0600 (Sun, 08 Nov 2009) | 1 line simplify condition ........ r76250 | benjamin.peterson | 2009-11-13 16:56:48 -0600 (Fri, 13 Nov 2009) | 1 line fix handling of a utf-8 bom #7313 ........ r76252 | benjamin.peterson | 2009-11-13 16:58:36 -0600 (Fri, 13 Nov 2009) | 1 line remove pdb turd ........ r76447 | benjamin.peterson | 2009-11-22 18:17:40 -0600 (Sun, 22 Nov 2009) | 1 line #7375 fix nested transformations in fix_urllib ........ r76506 | benjamin.peterson | 2009-11-24 18:34:31 -0600 (Tue, 24 Nov 2009) | 1 line use generator expressions in any() ........ ................ ................ --- Doc/library/calendar.rst | 4 +-- Doc/library/socketserver.rst | 6 ++-- Doc/library/string.rst | 9 +++--- Doc/library/xml.dom.rst | 2 +- Doc/library/zipfile.rst | 2 +- Lib/lib2to3/fixes/fix_imports.py | 2 +- Lib/lib2to3/fixes/fix_next.py | 2 +- Lib/lib2to3/fixes/fix_renames.py | 2 +- Lib/lib2to3/fixes/fix_urllib.py | 10 +++--- Lib/lib2to3/main.py | 11 +++---- Lib/lib2to3/pgen2/tokenize.py | 10 ++++-- Lib/lib2to3/refactor.py | 49 ++++++++++++++---------------- Lib/lib2to3/tests/data/bom.py | 3 ++ Lib/lib2to3/tests/test_fixers.py | 15 +++++++-- Lib/lib2to3/tests/test_refactor.py | 20 +++++++----- 15 files changed, 82 insertions(+), 65 deletions(-) create mode 100644 Lib/lib2to3/tests/data/bom.py diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 1cf958cba7d..f8c14fa0d4e 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -123,7 +123,7 @@ it's the base calendar for all computations. Print a month's calendar as returned by :meth:`formatmonth`. - .. method:: formatyear(theyear, themonth, w=2, l=1, c=6, m=3) + .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) Return a *m*-column calendar for an entire year as a multi-line string. Optional parameters *w*, *l*, and *c* are for date column width, lines per @@ -152,7 +152,7 @@ it's the base calendar for all computations. used. - .. method:: formatyear(theyear, themonth, width=3) + .. method:: formatyear(theyear, width=3) Return a year's calendar as an HTML table. *width* (defaulting to 3) specifies the number of months per row. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 9e0e9260c39..82d1107b120 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -447,9 +447,9 @@ This is the server side:: socket.sendto(data.upper(), self.client_address) if __name__ == "__main__": - HOST, PORT = "localhost", 9999 - server = socketserver.UDPServer((HOST, PORT), MyUDPHandler) - server.serve_forever() + HOST, PORT = "localhost", 9999 + server = socketserver.UDPServer((HOST, PORT), MyUDPHandler) + server.serve_forever() This is the client side:: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 2f7641aa616..93791c7569d 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -521,13 +521,12 @@ these rules. The methods of :class:`Template` are: templates containing dangling delimiters, unmatched braces, or placeholders that are not valid Python identifiers. -:class:`Template` instances also provide one public data attribute: + :class:`Template` instances also provide one public data attribute: + .. attribute:: template -.. attribute:: string.template - - This is the object passed to the constructor's *template* argument. In general, - you shouldn't change it, but read-only access is not enforced. + This is the object passed to the constructor's *template* argument. In + general, you shouldn't change it, but read-only access is not enforced. Here is an example of how to use a Template: diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index fe21804235d..45d30d67299 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -76,7 +76,7 @@ implementations are free to support the strict mapping from IDL). See section `Document Object Model (DOM) Level 1 Specification `_ The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - `Python Language Mapping Specification `_ + `Python Language Mapping Specification `_ This specifies the mapping from OMG IDL to Python. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index ae8751e60ae..7e9f4d841b1 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -19,7 +19,7 @@ documentation). It can handle ZIP files that use the ZIP64 extensions (that is ZIP files that are more than 4 GByte in size). It supports decryption of encrypted files in ZIP archives, but it currently cannot create an encrypted file. Decryption is extremely slow as it is -implemented in native python rather than C. +implemented in native Python rather than C. For other archive formats, see the :mod:`bz2`, :mod:`gzip`, and :mod:`tarfile` modules. diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py index 6c41c718fb1..5c053c0fde0 100644 --- a/Lib/lib2to3/fixes/fix_imports.py +++ b/Lib/lib2to3/fixes/fix_imports.py @@ -108,7 +108,7 @@ class FixImports(fixer_base.BaseFix): # Module usage could be in the trailer of an attribute lookup, so we # might have nested matches when "bare_with_attr" is present. if "bare_with_attr" not in results and \ - any([match(obj) for obj in attr_chain(node, "parent")]): + any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py index c999d6f71e2..66a00c27b65 100644 --- a/Lib/lib2to3/fixes/fix_next.py +++ b/Lib/lib2to3/fixes/fix_next.py @@ -99,4 +99,4 @@ def find_assign(node): def is_subtree(root, node): if root == node: return True - return any([is_subtree(c, node) for c in root.children]) + return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py index 275e23f322f..c9f68d5d768 100644 --- a/Lib/lib2to3/fixes/fix_renames.py +++ b/Lib/lib2to3/fixes/fix_renames.py @@ -49,7 +49,7 @@ class FixRenames(fixer_base.BaseFix): match = super(FixRenames, self).match results = match(node) if results: - if any([match(obj) for obj in attr_chain(node, "parent")]): + if any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index d11220c5e18..db18ca84eef 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -63,7 +63,8 @@ def build_pattern(): yield """import_name< 'import' dotted_as_name< module_as=%r 'as' any > > """ % old_module - yield """power< module_dot=%r trailer< '.' member=%s > any* > + # bare_with_attr has a special significance for FixImports.match(). + yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > """ % (old_module, members) @@ -150,12 +151,11 @@ class FixUrllib(FixImports): def transform_dot(self, node, results): """Transform for calls to module members in code.""" - module_dot = results.get('module_dot') + module_dot = results.get('bare_with_attr') member = results.get('member') - # this may be a list of length one, or just a node + new_name = None if isinstance(member, list): member = member[0] - new_name = None for change in MAPPING[module_dot.value]: if member.value in change[1]: new_name = change[0] @@ -171,7 +171,7 @@ class FixUrllib(FixImports): self.transform_import(node, results) elif results.get('mod_member'): self.transform_member(node, results) - elif results.get('module_dot'): + elif results.get('bare_with_attr'): self.transform_dot(node, results) # Renaming and star imports are not supported for these modules. elif results.get('module_star'): diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py index 7c26cdbccc8..6e096933efc 100644 --- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -90,8 +90,7 @@ def main(fixer_pkg, args=None): parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations (fixes/fix_*.py)") parser.add_option("-p", "--print-function", action="store_true", - help="DEPRECATED Modify the grammar so that print() is " - "a function") + help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", @@ -103,12 +102,10 @@ def main(fixer_pkg, args=None): # Parse command line arguments refactor_stdin = False + flags = {} options, args = parser.parse_args(args) if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") - if options.print_function: - warn("-p is deprecated; " - "detection of from __future__ import print_function is automatic") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if options.list_fixes: @@ -126,6 +123,8 @@ def main(fixer_pkg, args=None): if options.write: print("Can't write to stdin.", file=sys.stderr) return 2 + if options.print_function: + flags["print_function"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO @@ -146,7 +145,7 @@ def main(fixer_pkg, args=None): else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit), + rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 4585ca3f2b3..7ae02801ea4 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -283,9 +283,13 @@ def detect_encoding(readline): # This behaviour mimics the Python interpreter raise SyntaxError("unknown encoding: " + encoding) - if bom_found and codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') + if bom_found: + if codec.name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + else: + # Allow it to be properly encoded and decoded. + encoding = 'utf-8-sig' return encoding first = read_or_stop() diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 5edf584548c..8bd61adb7fc 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -18,7 +18,6 @@ import logging import operator import collections import io -import warnings from itertools import chain # Local imports @@ -139,26 +138,23 @@ def _detect_future_print(source): if have_docstring: break have_docstring = True - elif tp == token.NAME: - if value == "from": - tp, value = advance() - if tp != token.NAME and value != "__future__": - break - tp, value = advance() - if tp != token.NAME and value != "import": - break - tp, value = advance() - if tp == token.OP and value == "(": - tp, value = advance() - while tp == token.NAME: - if value == "print_function": - return True - tp, value = advance() - if tp != token.OP and value != ",": - break - tp, value = advance() - else: + elif tp == token.NAME and value == "from": + tp, value = advance() + if tp != token.NAME and value != "__future__": break + tp, value = advance() + if tp != token.NAME and value != "import": + break + tp, value = advance() + if tp == token.OP and value == "(": + tp, value = advance() + while tp == token.NAME: + if value == "print_function": + return True + tp, value = advance() + if tp != token.OP and value != ",": + break + tp, value = advance() else: break except StopIteration: @@ -172,7 +168,7 @@ class FixerError(Exception): class RefactoringTool(object): - _default_options = {} + _default_options = {"print_function" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -189,15 +185,16 @@ class RefactoringTool(object): self.explicit = explicit or [] self.options = self._default_options.copy() if options is not None: - if "print_function" in options: - warnings.warn("the 'print_function' option is deprecated", - DeprecationWarning) self.options.update(options) + if self.options["print_function"]: + self.grammar = pygram.python_grammar_no_print_statement + else: + self.grammar = pygram.python_grammar self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] self.wrote = False - self.driver = driver.Driver(pygram.python_grammar, + self.driver = driver.Driver(self.grammar, convert=pytree.convert, logger=self.logger) self.pre_order, self.post_order = self.get_fixers() @@ -353,7 +350,7 @@ class RefactoringTool(object): name, err.__class__.__name__, err) return finally: - self.driver.grammar = pygram.python_grammar + self.driver.grammar = self.grammar self.log_debug("Refactoring %s", name) self.refactor_tree(tree, name) return tree diff --git a/Lib/lib2to3/tests/data/bom.py b/Lib/lib2to3/tests/data/bom.py new file mode 100644 index 00000000000..ecb782a30d4 --- /dev/null +++ b/Lib/lib2to3/tests/data/bom.py @@ -0,0 +1,3 @@ +# coding: utf-8 +print "BOM BOOM!" + diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 2a393596ef1..ea361dec1f2 100755 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -1753,6 +1753,8 @@ class Test_urllib(FixerTestCase): for old, changes in self.modules.items(): for new, members in changes: for member in members: + new_import = ", ".join([n for (n, mems) + in self.modules[old]]) b = """ import %s foo(%s.%s) @@ -1760,9 +1762,16 @@ class Test_urllib(FixerTestCase): a = """ import %s foo(%s.%s) - """ % (", ".join([n for (n, mems) - in self.modules[old]]), - new, member) + """ % (new_import, new, member) + self.check(b, a) + b = """ + import %s + %s.%s(%s.%s) + """ % (old, old, member, old, member) + a = """ + import %s + %s.%s(%s.%s) + """ % (new_import, new, member, new, member) self.check(b, a) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 5ba23a5b3b3..2263f50eeb7 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -4,6 +4,7 @@ Unit tests for refactor.py. import sys import os +import codecs import operator import io import tempfile @@ -45,12 +46,10 @@ class TestRefactoringTool(unittest.TestCase): return refactor.RefactoringTool(fixers, options, explicit) def test_print_function_option(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True}) - self.assertEqual(len(w), 1) - msg, = w - self.assertTrue(msg.category is DeprecationWarning) + rt = self.rt({"print_function" : True}) + self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement) + self.assertTrue(rt.driver.grammar is + pygram.python_grammar_no_print_statement) def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] @@ -179,10 +178,12 @@ from __future__ import print_function""" try: rt.refactor_file(test_file, True) - self.assertNotEqual(old_contents, read_file()) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) finally: with open(test_file, "wb") as fp: fp.write(old_contents) + return new_contents def test_refactor_file(self): test_file = os.path.join(FIXER_DIR, "parrot_example.py") @@ -223,6 +224,11 @@ from __future__ import print_function""" fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_bom(self): + fn = os.path.join(TEST_DATA_DIR, "bom.py") + data = self.check_file_refactoring(fn) + self.assertTrue(data.startswith(codecs.BOM_UTF8)) + def test_crlf_newlines(self): old_sep = os.linesep os.linesep = "\r\n"