cpython/Lib/lib2to3/patcomp.py

202 lines
6.8 KiB
Python
Raw Normal View History

2008-03-19 01:43:46 -03:00
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Pattern compiler.
The grammer is taken from PatternGrammar.txt.
The compiler compiles a pattern to a pytree.*Pattern instance.
"""
__author__ = "Guido van Rossum <guido@python.org>"
# Python imports
import os
# Fairly local imports
Merged revisions 73771,73811,73840,73842,73848-73849,73861,73957-73960,73964-73969,73972-73974,73977,73981,73984,74065,74113 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r73771 | benjamin.peterson | 2009-07-02 10:56:55 -0500 (Thu, 02 Jul 2009) | 1 line force the imports fixer to be run after the import one #6400 ........ r73811 | benjamin.peterson | 2009-07-03 09:03:14 -0500 (Fri, 03 Jul 2009) | 1 line check for sep, not pathsep when looking for a subpackage #6408 ........ r73840 | benjamin.peterson | 2009-07-04 09:52:28 -0500 (Sat, 04 Jul 2009) | 1 line don't print diffs by default; it's annoying ........ r73842 | benjamin.peterson | 2009-07-04 09:58:46 -0500 (Sat, 04 Jul 2009) | 1 line complain when not showing diffs or writing ........ r73848 | alexandre.vassalotti | 2009-07-04 23:38:19 -0500 (Sat, 04 Jul 2009) | 2 lines Fix test_refactor_stdin to handle print_output() method with 4 arguments. ........ r73849 | alexandre.vassalotti | 2009-07-04 23:43:18 -0500 (Sat, 04 Jul 2009) | 5 lines Issue 2370: Add fixer for the removal of operator.isCallable() and operator.sequenceIncludes(). Patch contributed by Jeff Balogh (and updated by me). ........ r73861 | benjamin.peterson | 2009-07-05 09:15:53 -0500 (Sun, 05 Jul 2009) | 1 line cleanup and use unicode where appropiate ........ r73957 | benjamin.peterson | 2009-07-11 15:49:56 -0500 (Sat, 11 Jul 2009) | 1 line fix calls to str() with unicode() ........ r73958 | benjamin.peterson | 2009-07-11 15:51:51 -0500 (Sat, 11 Jul 2009) | 1 line more str() -> unicode() ........ r73959 | benjamin.peterson | 2009-07-11 16:40:08 -0500 (Sat, 11 Jul 2009) | 1 line add tests for refactor_dir() ........ r73960 | benjamin.peterson | 2009-07-11 16:44:32 -0500 (Sat, 11 Jul 2009) | 1 line don't parse files just because they end with 'py' (no dot) ........ r73964 | benjamin.peterson | 2009-07-11 17:30:15 -0500 (Sat, 11 Jul 2009) | 1 line simplify ........ r73965 | benjamin.peterson | 2009-07-11 17:31:30 -0500 (Sat, 11 Jul 2009) | 1 line remove usage of get_prefix() ........ r73966 | benjamin.peterson | 2009-07-11 17:33:35 -0500 (Sat, 11 Jul 2009) | 1 line revert unintended change in 73965 ........ r73967 | benjamin.peterson | 2009-07-11 17:34:44 -0500 (Sat, 11 Jul 2009) | 1 line avoid expensive checks and assume the node did change ........ r73968 | benjamin.peterson | 2009-07-11 20:46:46 -0500 (Sat, 11 Jul 2009) | 1 line use a regular dict for the heads to avoid adding lists in the loop ........ r73969 | benjamin.peterson | 2009-07-11 20:50:43 -0500 (Sat, 11 Jul 2009) | 1 line prefix headnode functions with '_' ........ r73972 | benjamin.peterson | 2009-07-11 21:25:45 -0500 (Sat, 11 Jul 2009) | 1 line try to make the head node dict as sparse as possible ........ r73973 | benjamin.peterson | 2009-07-11 21:59:49 -0500 (Sat, 11 Jul 2009) | 1 line a better idea; add an option to *not* print diffs ........ r73974 | benjamin.peterson | 2009-07-11 22:00:29 -0500 (Sat, 11 Jul 2009) | 1 line add space ........ r73977 | benjamin.peterson | 2009-07-12 10:16:07 -0500 (Sun, 12 Jul 2009) | 1 line update get_headnode_dict tests for recent changes ........ r73981 | benjamin.peterson | 2009-07-12 12:06:39 -0500 (Sun, 12 Jul 2009) | 4 lines detect when "from __future__ import print_function" is given Deprecate the 'print_function' option and the -p flag ........ r73984 | benjamin.peterson | 2009-07-12 16:16:37 -0500 (Sun, 12 Jul 2009) | 1 line add tests for Call; thanks Joe Amenta ........ r74065 | benjamin.peterson | 2009-07-17 12:52:49 -0500 (Fri, 17 Jul 2009) | 1 line pathname2url and url2pathname are in urllib.request not urllib.parse #6496 ........ r74113 | benjamin.peterson | 2009-07-20 08:56:57 -0500 (Mon, 20 Jul 2009) | 1 line fix deprecation warnings in tests ........
2009-07-20 12:33:09 -03:00
from .pgen2 import driver, literals, token, tokenize, parse, grammar
2008-03-19 01:43:46 -03:00
# Really local imports
from . import pytree
from . import pygram
# The pattern grammar file
_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
"PatternGrammar.txt")
Merged revisions 72523,72950-72951,72994,73003,73033,73036-73040,73091-73093,73096,73179-73181,73192,73231,73244,73255-73256,73365 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r72523 | benjamin.peterson | 2009-05-09 14:42:26 -0500 (Sat, 09 May 2009) | 1 line remove parenthesis ........ r72950 | benjamin.peterson | 2009-05-26 18:19:45 -0500 (Tue, 26 May 2009) | 1 line remove unused imports ........ r72951 | benjamin.peterson | 2009-05-26 18:27:00 -0500 (Tue, 26 May 2009) | 1 line this is no longer executable ........ r72994 | benjamin.peterson | 2009-05-28 15:32:54 -0500 (Thu, 28 May 2009) | 1 line fix test_all_fixers on Windows #6134 ........ r73003 | benjamin.peterson | 2009-05-28 21:57:28 -0500 (Thu, 28 May 2009) | 4 lines make 2to3 test utilities easier to use with other applications (3to2) Patch by Joe Amenta ........ r73033 | benjamin.peterson | 2009-05-29 16:58:32 -0500 (Fri, 29 May 2009) | 1 line update grammar for multi with statement ........ r73036 | benjamin.peterson | 2009-05-29 17:33:20 -0500 (Fri, 29 May 2009) | 1 line simplify fix_unicode ........ r73037 | benjamin.peterson | 2009-05-29 17:53:03 -0500 (Fri, 29 May 2009) | 1 line add custom error for pattern syntax errors ........ r73038 | benjamin.peterson | 2009-05-29 17:55:00 -0500 (Fri, 29 May 2009) | 1 line complain if details are attached to a token ........ r73039 | benjamin.peterson | 2009-05-29 18:00:28 -0500 (Fri, 29 May 2009) | 1 line add a test for whitespace ........ r73040 | benjamin.peterson | 2009-05-29 18:01:17 -0500 (Fri, 29 May 2009) | 1 line a fix for emacs highlighting ........ r73091 | benjamin.peterson | 2009-05-31 20:55:25 -0500 (Sun, 31 May 2009) | 1 line deprecate set_prefix() and get_prefix() in favor of a prefix property ........ r73092 | benjamin.peterson | 2009-05-31 21:00:51 -0500 (Sun, 31 May 2009) | 1 line change hideous java naming scheme ........ r73093 | benjamin.peterson | 2009-05-31 21:01:39 -0500 (Sun, 31 May 2009) | 1 line remove dated comment ........ r73096 | benjamin.peterson | 2009-05-31 21:40:53 -0500 (Sun, 31 May 2009) | 1 line group tests ........ r73179 | benjamin.peterson | 2009-06-03 13:09:53 -0500 (Wed, 03 Jun 2009) | 1 line handle the case where there's multiple trailers #6185 ........ r73180 | benjamin.peterson | 2009-06-03 13:18:05 -0500 (Wed, 03 Jun 2009) | 1 line scrap __main__ section ........ r73181 | benjamin.peterson | 2009-06-03 13:24:48 -0500 (Wed, 03 Jun 2009) | 1 line remove shebang lines and __main__ sections ........ r73192 | benjamin.peterson | 2009-06-03 19:16:30 -0500 (Wed, 03 Jun 2009) | 4 lines actually test something here Thanks to Joe Amenta for noticing.y ........ r73231 | benjamin.peterson | 2009-06-04 13:38:50 -0500 (Thu, 04 Jun 2009) | 1 line remove unused variable ........ r73244 | benjamin.peterson | 2009-06-05 08:39:25 -0500 (Fri, 05 Jun 2009) | 1 line allow fixers to give different options in setUp ........ r73255 | benjamin.peterson | 2009-06-06 11:23:46 -0500 (Sat, 06 Jun 2009) | 1 line fix the except fixer on one line suites #6222 ........ r73256 | benjamin.peterson | 2009-06-06 11:27:40 -0500 (Sat, 06 Jun 2009) | 1 line test one-line else and finally clauses ........ r73365 | benjamin.peterson | 2009-06-11 17:01:32 -0500 (Thu, 11 Jun 2009) | 1 line normalize whitespace ........
2009-06-11 19:06:46 -03:00
class PatternSyntaxError(Exception):
pass
2008-03-19 01:43:46 -03:00
def tokenize_wrapper(input):
"""Tokenizes a string suppressing significant whitespace."""
Merged revisions 68503,68507,68694,69054,69673,69679-69681,70991,70999,71003,71695 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r68503 | benjamin.peterson | 2009-01-10 14:14:49 -0600 (Sat, 10 Jan 2009) | 1 line use variable ........ r68507 | benjamin.peterson | 2009-01-10 15:13:16 -0600 (Sat, 10 Jan 2009) | 1 line rewrap ........ r68694 | benjamin.peterson | 2009-01-17 17:55:59 -0600 (Sat, 17 Jan 2009) | 1 line test for specific node type ........ r69054 | guilherme.polo | 2009-01-28 10:01:54 -0600 (Wed, 28 Jan 2009) | 2 lines Added mapping for the ttk module. ........ r69673 | benjamin.peterson | 2009-02-16 09:38:22 -0600 (Mon, 16 Feb 2009) | 1 line fix handling of as imports #5279 ........ r69679 | benjamin.peterson | 2009-02-16 11:36:06 -0600 (Mon, 16 Feb 2009) | 1 line make Base.get_next_sibling() and Base.get_prev_sibling() properties ........ r69680 | benjamin.peterson | 2009-02-16 11:41:48 -0600 (Mon, 16 Feb 2009) | 1 line normalize docstrings in pytree according to PEP 11 ........ r69681 | benjamin.peterson | 2009-02-16 11:43:09 -0600 (Mon, 16 Feb 2009) | 1 line use a set ........ r70991 | benjamin.peterson | 2009-04-01 15:54:50 -0500 (Wed, 01 Apr 2009) | 1 line map urllib.urlopen to urllib.request.open #5637 ........ r70999 | benjamin.peterson | 2009-04-01 17:36:47 -0500 (Wed, 01 Apr 2009) | 1 line add very alpha support to 2to3 for running concurrently with multiprocessing ........ r71003 | benjamin.peterson | 2009-04-01 18:10:43 -0500 (Wed, 01 Apr 2009) | 1 line fix when multiprocessing is not available or used ........ r71695 | benjamin.peterson | 2009-04-17 22:21:29 -0500 (Fri, 17 Apr 2009) | 1 line refactor multiprocessing support, so it's less hacky to employ and only loads mp when needed ........
2009-05-05 20:13:58 -03:00
skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
2008-03-19 01:43:46 -03:00
tokens = tokenize.generate_tokens(driver.generate_lines(input).next)
for quintuple in tokens:
type, value, start, end, line_text = quintuple
if type not in skip:
yield quintuple
class PatternCompiler(object):
def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE):
"""Initializer.
Takes an optional alternative filename for the pattern grammar.
"""
self.grammar = driver.load_grammar(grammar_file)
self.syms = pygram.Symbols(self.grammar)
self.pygrammar = pygram.python_grammar
self.pysyms = pygram.python_symbols
self.driver = driver.Driver(self.grammar, convert=pattern_convert)
def compile_pattern(self, input, debug=False):
"""Compiles a pattern string to a nested pytree.*Pattern object."""
tokens = tokenize_wrapper(input)
Merged revisions 72523,72950-72951,72994,73003,73033,73036-73040,73091-73093,73096,73179-73181,73192,73231,73244,73255-73256,73365 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r72523 | benjamin.peterson | 2009-05-09 14:42:26 -0500 (Sat, 09 May 2009) | 1 line remove parenthesis ........ r72950 | benjamin.peterson | 2009-05-26 18:19:45 -0500 (Tue, 26 May 2009) | 1 line remove unused imports ........ r72951 | benjamin.peterson | 2009-05-26 18:27:00 -0500 (Tue, 26 May 2009) | 1 line this is no longer executable ........ r72994 | benjamin.peterson | 2009-05-28 15:32:54 -0500 (Thu, 28 May 2009) | 1 line fix test_all_fixers on Windows #6134 ........ r73003 | benjamin.peterson | 2009-05-28 21:57:28 -0500 (Thu, 28 May 2009) | 4 lines make 2to3 test utilities easier to use with other applications (3to2) Patch by Joe Amenta ........ r73033 | benjamin.peterson | 2009-05-29 16:58:32 -0500 (Fri, 29 May 2009) | 1 line update grammar for multi with statement ........ r73036 | benjamin.peterson | 2009-05-29 17:33:20 -0500 (Fri, 29 May 2009) | 1 line simplify fix_unicode ........ r73037 | benjamin.peterson | 2009-05-29 17:53:03 -0500 (Fri, 29 May 2009) | 1 line add custom error for pattern syntax errors ........ r73038 | benjamin.peterson | 2009-05-29 17:55:00 -0500 (Fri, 29 May 2009) | 1 line complain if details are attached to a token ........ r73039 | benjamin.peterson | 2009-05-29 18:00:28 -0500 (Fri, 29 May 2009) | 1 line add a test for whitespace ........ r73040 | benjamin.peterson | 2009-05-29 18:01:17 -0500 (Fri, 29 May 2009) | 1 line a fix for emacs highlighting ........ r73091 | benjamin.peterson | 2009-05-31 20:55:25 -0500 (Sun, 31 May 2009) | 1 line deprecate set_prefix() and get_prefix() in favor of a prefix property ........ r73092 | benjamin.peterson | 2009-05-31 21:00:51 -0500 (Sun, 31 May 2009) | 1 line change hideous java naming scheme ........ r73093 | benjamin.peterson | 2009-05-31 21:01:39 -0500 (Sun, 31 May 2009) | 1 line remove dated comment ........ r73096 | benjamin.peterson | 2009-05-31 21:40:53 -0500 (Sun, 31 May 2009) | 1 line group tests ........ r73179 | benjamin.peterson | 2009-06-03 13:09:53 -0500 (Wed, 03 Jun 2009) | 1 line handle the case where there's multiple trailers #6185 ........ r73180 | benjamin.peterson | 2009-06-03 13:18:05 -0500 (Wed, 03 Jun 2009) | 1 line scrap __main__ section ........ r73181 | benjamin.peterson | 2009-06-03 13:24:48 -0500 (Wed, 03 Jun 2009) | 1 line remove shebang lines and __main__ sections ........ r73192 | benjamin.peterson | 2009-06-03 19:16:30 -0500 (Wed, 03 Jun 2009) | 4 lines actually test something here Thanks to Joe Amenta for noticing.y ........ r73231 | benjamin.peterson | 2009-06-04 13:38:50 -0500 (Thu, 04 Jun 2009) | 1 line remove unused variable ........ r73244 | benjamin.peterson | 2009-06-05 08:39:25 -0500 (Fri, 05 Jun 2009) | 1 line allow fixers to give different options in setUp ........ r73255 | benjamin.peterson | 2009-06-06 11:23:46 -0500 (Sat, 06 Jun 2009) | 1 line fix the except fixer on one line suites #6222 ........ r73256 | benjamin.peterson | 2009-06-06 11:27:40 -0500 (Sat, 06 Jun 2009) | 1 line test one-line else and finally clauses ........ r73365 | benjamin.peterson | 2009-06-11 17:01:32 -0500 (Thu, 11 Jun 2009) | 1 line normalize whitespace ........
2009-06-11 19:06:46 -03:00
try:
root = self.driver.parse_tokens(tokens, debug=debug)
Merged revisions 79911,79916-79917,80018,80418,80572-80573,80635-80639,80668,80922 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r79911 | benjamin.peterson | 2010-04-09 15:38:53 -0500 (Fri, 09 Apr 2010) | 1 line use absolute import ........ r79916 | benjamin.peterson | 2010-04-09 16:05:21 -0500 (Fri, 09 Apr 2010) | 1 line generalize detection of __future__ imports and attach them to the tree ........ r79917 | benjamin.peterson | 2010-04-09 16:11:44 -0500 (Fri, 09 Apr 2010) | 1 line don't try to 'fix' relative imports when absolute_import is enabled #8858 ........ r80018 | benjamin.peterson | 2010-04-12 16:12:12 -0500 (Mon, 12 Apr 2010) | 4 lines prevent diffs from being mangled is multiprocess mode #6409 Patch by George Boutsioukis. ........ r80418 | benjamin.peterson | 2010-04-23 16:00:03 -0500 (Fri, 23 Apr 2010) | 1 line remove unhelpful description ........ r80572 | benjamin.peterson | 2010-04-27 20:33:54 -0500 (Tue, 27 Apr 2010) | 1 line use unicode literals ........ r80573 | jeffrey.yasskin | 2010-04-27 23:08:27 -0500 (Tue, 27 Apr 2010) | 6 lines Don't transform imports that are already relative. 2to3 turned from . import refactor into from .. import refactor which broke the transformation of 2to3 itself. ........ r80635 | benjamin.peterson | 2010-04-29 16:02:23 -0500 (Thu, 29 Apr 2010) | 1 line remove imports ........ r80636 | benjamin.peterson | 2010-04-29 16:02:41 -0500 (Thu, 29 Apr 2010) | 1 line unicode literal ........ r80637 | benjamin.peterson | 2010-04-29 16:03:42 -0500 (Thu, 29 Apr 2010) | 1 line must pass a string to Number ........ r80638 | benjamin.peterson | 2010-04-29 16:05:34 -0500 (Thu, 29 Apr 2010) | 1 line unicode literals ........ r80639 | benjamin.peterson | 2010-04-29 16:06:09 -0500 (Thu, 29 Apr 2010) | 1 line pass string to Number ........ r80668 | jeffrey.yasskin | 2010-04-30 18:02:47 -0500 (Fri, 30 Apr 2010) | 4 lines Make 2to3 run under Python 2.5 so that the benchmark suite at http://hg.python.org/benchmarks/ can use it and still run on implementations that haven't gotten to 2.6 yet. Fixes issue 8566. ........ r80922 | benjamin.peterson | 2010-05-07 11:06:25 -0500 (Fri, 07 May 2010) | 1 line prevent xrange transformation from wrapping range calls it produces in list ........
2010-05-07 15:58:23 -03:00
except parse.ParseError, e:
Merged revisions 72523,72950-72951,72994,73003,73033,73036-73040,73091-73093,73096,73179-73181,73192,73231,73244,73255-73256,73365 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r72523 | benjamin.peterson | 2009-05-09 14:42:26 -0500 (Sat, 09 May 2009) | 1 line remove parenthesis ........ r72950 | benjamin.peterson | 2009-05-26 18:19:45 -0500 (Tue, 26 May 2009) | 1 line remove unused imports ........ r72951 | benjamin.peterson | 2009-05-26 18:27:00 -0500 (Tue, 26 May 2009) | 1 line this is no longer executable ........ r72994 | benjamin.peterson | 2009-05-28 15:32:54 -0500 (Thu, 28 May 2009) | 1 line fix test_all_fixers on Windows #6134 ........ r73003 | benjamin.peterson | 2009-05-28 21:57:28 -0500 (Thu, 28 May 2009) | 4 lines make 2to3 test utilities easier to use with other applications (3to2) Patch by Joe Amenta ........ r73033 | benjamin.peterson | 2009-05-29 16:58:32 -0500 (Fri, 29 May 2009) | 1 line update grammar for multi with statement ........ r73036 | benjamin.peterson | 2009-05-29 17:33:20 -0500 (Fri, 29 May 2009) | 1 line simplify fix_unicode ........ r73037 | benjamin.peterson | 2009-05-29 17:53:03 -0500 (Fri, 29 May 2009) | 1 line add custom error for pattern syntax errors ........ r73038 | benjamin.peterson | 2009-05-29 17:55:00 -0500 (Fri, 29 May 2009) | 1 line complain if details are attached to a token ........ r73039 | benjamin.peterson | 2009-05-29 18:00:28 -0500 (Fri, 29 May 2009) | 1 line add a test for whitespace ........ r73040 | benjamin.peterson | 2009-05-29 18:01:17 -0500 (Fri, 29 May 2009) | 1 line a fix for emacs highlighting ........ r73091 | benjamin.peterson | 2009-05-31 20:55:25 -0500 (Sun, 31 May 2009) | 1 line deprecate set_prefix() and get_prefix() in favor of a prefix property ........ r73092 | benjamin.peterson | 2009-05-31 21:00:51 -0500 (Sun, 31 May 2009) | 1 line change hideous java naming scheme ........ r73093 | benjamin.peterson | 2009-05-31 21:01:39 -0500 (Sun, 31 May 2009) | 1 line remove dated comment ........ r73096 | benjamin.peterson | 2009-05-31 21:40:53 -0500 (Sun, 31 May 2009) | 1 line group tests ........ r73179 | benjamin.peterson | 2009-06-03 13:09:53 -0500 (Wed, 03 Jun 2009) | 1 line handle the case where there's multiple trailers #6185 ........ r73180 | benjamin.peterson | 2009-06-03 13:18:05 -0500 (Wed, 03 Jun 2009) | 1 line scrap __main__ section ........ r73181 | benjamin.peterson | 2009-06-03 13:24:48 -0500 (Wed, 03 Jun 2009) | 1 line remove shebang lines and __main__ sections ........ r73192 | benjamin.peterson | 2009-06-03 19:16:30 -0500 (Wed, 03 Jun 2009) | 4 lines actually test something here Thanks to Joe Amenta for noticing.y ........ r73231 | benjamin.peterson | 2009-06-04 13:38:50 -0500 (Thu, 04 Jun 2009) | 1 line remove unused variable ........ r73244 | benjamin.peterson | 2009-06-05 08:39:25 -0500 (Fri, 05 Jun 2009) | 1 line allow fixers to give different options in setUp ........ r73255 | benjamin.peterson | 2009-06-06 11:23:46 -0500 (Sat, 06 Jun 2009) | 1 line fix the except fixer on one line suites #6222 ........ r73256 | benjamin.peterson | 2009-06-06 11:27:40 -0500 (Sat, 06 Jun 2009) | 1 line test one-line else and finally clauses ........ r73365 | benjamin.peterson | 2009-06-11 17:01:32 -0500 (Thu, 11 Jun 2009) | 1 line normalize whitespace ........
2009-06-11 19:06:46 -03:00
raise PatternSyntaxError(str(e))
2008-03-19 01:43:46 -03:00
return self.compile_node(root)
def compile_node(self, node):
"""Compiles a node, recursively.
This is one big switch on the node type.
"""
# XXX Optimize certain Wildcard-containing-Wildcard patterns
# that can be merged
if node.type == self.syms.Matcher:
node = node.children[0] # Avoid unneeded recursion
if node.type == self.syms.Alternatives:
# Skip the odd children since they are just '|' tokens
alts = [self.compile_node(ch) for ch in node.children[::2]]
if len(alts) == 1:
return alts[0]
p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1)
return p.optimize()
if node.type == self.syms.Alternative:
units = [self.compile_node(ch) for ch in node.children]
if len(units) == 1:
return units[0]
p = pytree.WildcardPattern([units], min=1, max=1)
return p.optimize()
if node.type == self.syms.NegatedUnit:
pattern = self.compile_basic(node.children[1:])
p = pytree.NegatedPattern(pattern)
return p.optimize()
assert node.type == self.syms.Unit
name = None
nodes = node.children
if len(nodes) >= 3 and nodes[1].type == token.EQUAL:
name = nodes[0].value
nodes = nodes[2:]
repeat = None
if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater:
repeat = nodes[-1]
nodes = nodes[:-1]
# Now we've reduced it to: STRING | NAME [Details] | (...) | [...]
pattern = self.compile_basic(nodes, repeat)
if repeat is not None:
assert repeat.type == self.syms.Repeater
children = repeat.children
child = children[0]
if child.type == token.STAR:
min = 0
max = pytree.HUGE
elif child.type == token.PLUS:
min = 1
max = pytree.HUGE
elif child.type == token.LBRACE:
assert children[-1].type == token.RBRACE
assert len(children) in (3, 5)
min = max = self.get_int(children[1])
if len(children) == 5:
max = self.get_int(children[3])
else:
assert False
if min != 1 or max != 1:
pattern = pattern.optimize()
pattern = pytree.WildcardPattern([[pattern]], min=min, max=max)
if name is not None:
pattern.name = name
return pattern.optimize()
def compile_basic(self, nodes, repeat=None):
# Compile STRING | NAME [Details] | (...) | [...]
assert len(nodes) >= 1
node = nodes[0]
if node.type == token.STRING:
value = unicode(literals.evalString(node.value))
Merged revisions 73771,73811,73840,73842,73848-73849,73861,73957-73960,73964-73969,73972-73974,73977,73981,73984,74065,74113 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r73771 | benjamin.peterson | 2009-07-02 10:56:55 -0500 (Thu, 02 Jul 2009) | 1 line force the imports fixer to be run after the import one #6400 ........ r73811 | benjamin.peterson | 2009-07-03 09:03:14 -0500 (Fri, 03 Jul 2009) | 1 line check for sep, not pathsep when looking for a subpackage #6408 ........ r73840 | benjamin.peterson | 2009-07-04 09:52:28 -0500 (Sat, 04 Jul 2009) | 1 line don't print diffs by default; it's annoying ........ r73842 | benjamin.peterson | 2009-07-04 09:58:46 -0500 (Sat, 04 Jul 2009) | 1 line complain when not showing diffs or writing ........ r73848 | alexandre.vassalotti | 2009-07-04 23:38:19 -0500 (Sat, 04 Jul 2009) | 2 lines Fix test_refactor_stdin to handle print_output() method with 4 arguments. ........ r73849 | alexandre.vassalotti | 2009-07-04 23:43:18 -0500 (Sat, 04 Jul 2009) | 5 lines Issue 2370: Add fixer for the removal of operator.isCallable() and operator.sequenceIncludes(). Patch contributed by Jeff Balogh (and updated by me). ........ r73861 | benjamin.peterson | 2009-07-05 09:15:53 -0500 (Sun, 05 Jul 2009) | 1 line cleanup and use unicode where appropiate ........ r73957 | benjamin.peterson | 2009-07-11 15:49:56 -0500 (Sat, 11 Jul 2009) | 1 line fix calls to str() with unicode() ........ r73958 | benjamin.peterson | 2009-07-11 15:51:51 -0500 (Sat, 11 Jul 2009) | 1 line more str() -> unicode() ........ r73959 | benjamin.peterson | 2009-07-11 16:40:08 -0500 (Sat, 11 Jul 2009) | 1 line add tests for refactor_dir() ........ r73960 | benjamin.peterson | 2009-07-11 16:44:32 -0500 (Sat, 11 Jul 2009) | 1 line don't parse files just because they end with 'py' (no dot) ........ r73964 | benjamin.peterson | 2009-07-11 17:30:15 -0500 (Sat, 11 Jul 2009) | 1 line simplify ........ r73965 | benjamin.peterson | 2009-07-11 17:31:30 -0500 (Sat, 11 Jul 2009) | 1 line remove usage of get_prefix() ........ r73966 | benjamin.peterson | 2009-07-11 17:33:35 -0500 (Sat, 11 Jul 2009) | 1 line revert unintended change in 73965 ........ r73967 | benjamin.peterson | 2009-07-11 17:34:44 -0500 (Sat, 11 Jul 2009) | 1 line avoid expensive checks and assume the node did change ........ r73968 | benjamin.peterson | 2009-07-11 20:46:46 -0500 (Sat, 11 Jul 2009) | 1 line use a regular dict for the heads to avoid adding lists in the loop ........ r73969 | benjamin.peterson | 2009-07-11 20:50:43 -0500 (Sat, 11 Jul 2009) | 1 line prefix headnode functions with '_' ........ r73972 | benjamin.peterson | 2009-07-11 21:25:45 -0500 (Sat, 11 Jul 2009) | 1 line try to make the head node dict as sparse as possible ........ r73973 | benjamin.peterson | 2009-07-11 21:59:49 -0500 (Sat, 11 Jul 2009) | 1 line a better idea; add an option to *not* print diffs ........ r73974 | benjamin.peterson | 2009-07-11 22:00:29 -0500 (Sat, 11 Jul 2009) | 1 line add space ........ r73977 | benjamin.peterson | 2009-07-12 10:16:07 -0500 (Sun, 12 Jul 2009) | 1 line update get_headnode_dict tests for recent changes ........ r73981 | benjamin.peterson | 2009-07-12 12:06:39 -0500 (Sun, 12 Jul 2009) | 4 lines detect when "from __future__ import print_function" is given Deprecate the 'print_function' option and the -p flag ........ r73984 | benjamin.peterson | 2009-07-12 16:16:37 -0500 (Sun, 12 Jul 2009) | 1 line add tests for Call; thanks Joe Amenta ........ r74065 | benjamin.peterson | 2009-07-17 12:52:49 -0500 (Fri, 17 Jul 2009) | 1 line pathname2url and url2pathname are in urllib.request not urllib.parse #6496 ........ r74113 | benjamin.peterson | 2009-07-20 08:56:57 -0500 (Mon, 20 Jul 2009) | 1 line fix deprecation warnings in tests ........
2009-07-20 12:33:09 -03:00
return pytree.LeafPattern(_type_of_literal(value), value)
2008-03-19 01:43:46 -03:00
elif node.type == token.NAME:
value = node.value
if value.isupper():
if value not in TOKEN_MAP:
Merged revisions 72523,72950-72951,72994,73003,73033,73036-73040,73091-73093,73096,73179-73181,73192,73231,73244,73255-73256,73365 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r72523 | benjamin.peterson | 2009-05-09 14:42:26 -0500 (Sat, 09 May 2009) | 1 line remove parenthesis ........ r72950 | benjamin.peterson | 2009-05-26 18:19:45 -0500 (Tue, 26 May 2009) | 1 line remove unused imports ........ r72951 | benjamin.peterson | 2009-05-26 18:27:00 -0500 (Tue, 26 May 2009) | 1 line this is no longer executable ........ r72994 | benjamin.peterson | 2009-05-28 15:32:54 -0500 (Thu, 28 May 2009) | 1 line fix test_all_fixers on Windows #6134 ........ r73003 | benjamin.peterson | 2009-05-28 21:57:28 -0500 (Thu, 28 May 2009) | 4 lines make 2to3 test utilities easier to use with other applications (3to2) Patch by Joe Amenta ........ r73033 | benjamin.peterson | 2009-05-29 16:58:32 -0500 (Fri, 29 May 2009) | 1 line update grammar for multi with statement ........ r73036 | benjamin.peterson | 2009-05-29 17:33:20 -0500 (Fri, 29 May 2009) | 1 line simplify fix_unicode ........ r73037 | benjamin.peterson | 2009-05-29 17:53:03 -0500 (Fri, 29 May 2009) | 1 line add custom error for pattern syntax errors ........ r73038 | benjamin.peterson | 2009-05-29 17:55:00 -0500 (Fri, 29 May 2009) | 1 line complain if details are attached to a token ........ r73039 | benjamin.peterson | 2009-05-29 18:00:28 -0500 (Fri, 29 May 2009) | 1 line add a test for whitespace ........ r73040 | benjamin.peterson | 2009-05-29 18:01:17 -0500 (Fri, 29 May 2009) | 1 line a fix for emacs highlighting ........ r73091 | benjamin.peterson | 2009-05-31 20:55:25 -0500 (Sun, 31 May 2009) | 1 line deprecate set_prefix() and get_prefix() in favor of a prefix property ........ r73092 | benjamin.peterson | 2009-05-31 21:00:51 -0500 (Sun, 31 May 2009) | 1 line change hideous java naming scheme ........ r73093 | benjamin.peterson | 2009-05-31 21:01:39 -0500 (Sun, 31 May 2009) | 1 line remove dated comment ........ r73096 | benjamin.peterson | 2009-05-31 21:40:53 -0500 (Sun, 31 May 2009) | 1 line group tests ........ r73179 | benjamin.peterson | 2009-06-03 13:09:53 -0500 (Wed, 03 Jun 2009) | 1 line handle the case where there's multiple trailers #6185 ........ r73180 | benjamin.peterson | 2009-06-03 13:18:05 -0500 (Wed, 03 Jun 2009) | 1 line scrap __main__ section ........ r73181 | benjamin.peterson | 2009-06-03 13:24:48 -0500 (Wed, 03 Jun 2009) | 1 line remove shebang lines and __main__ sections ........ r73192 | benjamin.peterson | 2009-06-03 19:16:30 -0500 (Wed, 03 Jun 2009) | 4 lines actually test something here Thanks to Joe Amenta for noticing.y ........ r73231 | benjamin.peterson | 2009-06-04 13:38:50 -0500 (Thu, 04 Jun 2009) | 1 line remove unused variable ........ r73244 | benjamin.peterson | 2009-06-05 08:39:25 -0500 (Fri, 05 Jun 2009) | 1 line allow fixers to give different options in setUp ........ r73255 | benjamin.peterson | 2009-06-06 11:23:46 -0500 (Sat, 06 Jun 2009) | 1 line fix the except fixer on one line suites #6222 ........ r73256 | benjamin.peterson | 2009-06-06 11:27:40 -0500 (Sat, 06 Jun 2009) | 1 line test one-line else and finally clauses ........ r73365 | benjamin.peterson | 2009-06-11 17:01:32 -0500 (Thu, 11 Jun 2009) | 1 line normalize whitespace ........
2009-06-11 19:06:46 -03:00
raise PatternSyntaxError("Invalid token: %r" % value)
if nodes[1:]:
raise PatternSyntaxError("Can't have details for token")
2008-03-19 01:43:46 -03:00
return pytree.LeafPattern(TOKEN_MAP[value])
else:
if value == "any":
type = None
elif not value.startswith("_"):
type = getattr(self.pysyms, value, None)
if type is None:
Merged revisions 72523,72950-72951,72994,73003,73033,73036-73040,73091-73093,73096,73179-73181,73192,73231,73244,73255-73256,73365 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r72523 | benjamin.peterson | 2009-05-09 14:42:26 -0500 (Sat, 09 May 2009) | 1 line remove parenthesis ........ r72950 | benjamin.peterson | 2009-05-26 18:19:45 -0500 (Tue, 26 May 2009) | 1 line remove unused imports ........ r72951 | benjamin.peterson | 2009-05-26 18:27:00 -0500 (Tue, 26 May 2009) | 1 line this is no longer executable ........ r72994 | benjamin.peterson | 2009-05-28 15:32:54 -0500 (Thu, 28 May 2009) | 1 line fix test_all_fixers on Windows #6134 ........ r73003 | benjamin.peterson | 2009-05-28 21:57:28 -0500 (Thu, 28 May 2009) | 4 lines make 2to3 test utilities easier to use with other applications (3to2) Patch by Joe Amenta ........ r73033 | benjamin.peterson | 2009-05-29 16:58:32 -0500 (Fri, 29 May 2009) | 1 line update grammar for multi with statement ........ r73036 | benjamin.peterson | 2009-05-29 17:33:20 -0500 (Fri, 29 May 2009) | 1 line simplify fix_unicode ........ r73037 | benjamin.peterson | 2009-05-29 17:53:03 -0500 (Fri, 29 May 2009) | 1 line add custom error for pattern syntax errors ........ r73038 | benjamin.peterson | 2009-05-29 17:55:00 -0500 (Fri, 29 May 2009) | 1 line complain if details are attached to a token ........ r73039 | benjamin.peterson | 2009-05-29 18:00:28 -0500 (Fri, 29 May 2009) | 1 line add a test for whitespace ........ r73040 | benjamin.peterson | 2009-05-29 18:01:17 -0500 (Fri, 29 May 2009) | 1 line a fix for emacs highlighting ........ r73091 | benjamin.peterson | 2009-05-31 20:55:25 -0500 (Sun, 31 May 2009) | 1 line deprecate set_prefix() and get_prefix() in favor of a prefix property ........ r73092 | benjamin.peterson | 2009-05-31 21:00:51 -0500 (Sun, 31 May 2009) | 1 line change hideous java naming scheme ........ r73093 | benjamin.peterson | 2009-05-31 21:01:39 -0500 (Sun, 31 May 2009) | 1 line remove dated comment ........ r73096 | benjamin.peterson | 2009-05-31 21:40:53 -0500 (Sun, 31 May 2009) | 1 line group tests ........ r73179 | benjamin.peterson | 2009-06-03 13:09:53 -0500 (Wed, 03 Jun 2009) | 1 line handle the case where there's multiple trailers #6185 ........ r73180 | benjamin.peterson | 2009-06-03 13:18:05 -0500 (Wed, 03 Jun 2009) | 1 line scrap __main__ section ........ r73181 | benjamin.peterson | 2009-06-03 13:24:48 -0500 (Wed, 03 Jun 2009) | 1 line remove shebang lines and __main__ sections ........ r73192 | benjamin.peterson | 2009-06-03 19:16:30 -0500 (Wed, 03 Jun 2009) | 4 lines actually test something here Thanks to Joe Amenta for noticing.y ........ r73231 | benjamin.peterson | 2009-06-04 13:38:50 -0500 (Thu, 04 Jun 2009) | 1 line remove unused variable ........ r73244 | benjamin.peterson | 2009-06-05 08:39:25 -0500 (Fri, 05 Jun 2009) | 1 line allow fixers to give different options in setUp ........ r73255 | benjamin.peterson | 2009-06-06 11:23:46 -0500 (Sat, 06 Jun 2009) | 1 line fix the except fixer on one line suites #6222 ........ r73256 | benjamin.peterson | 2009-06-06 11:27:40 -0500 (Sat, 06 Jun 2009) | 1 line test one-line else and finally clauses ........ r73365 | benjamin.peterson | 2009-06-11 17:01:32 -0500 (Thu, 11 Jun 2009) | 1 line normalize whitespace ........
2009-06-11 19:06:46 -03:00
raise PatternSyntaxError("Invalid symbol: %r" % value)
2008-03-19 01:43:46 -03:00
if nodes[1:]: # Details present
content = [self.compile_node(nodes[1].children[1])]
else:
content = None
return pytree.NodePattern(type, content)
elif node.value == "(":
return self.compile_node(nodes[1])
elif node.value == "[":
assert repeat is None
subpattern = self.compile_node(nodes[1])
return pytree.WildcardPattern([[subpattern]], min=0, max=1)
assert False, node
def get_int(self, node):
assert node.type == token.NUMBER
return int(node.value)
# Map named tokens to the type value for a LeafPattern
TOKEN_MAP = {"NAME": token.NAME,
"STRING": token.STRING,
"NUMBER": token.NUMBER,
"TOKEN": None}
Merged revisions 73771,73811,73840,73842,73848-73849,73861,73957-73960,73964-73969,73972-73974,73977,73981,73984,74065,74113 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r73771 | benjamin.peterson | 2009-07-02 10:56:55 -0500 (Thu, 02 Jul 2009) | 1 line force the imports fixer to be run after the import one #6400 ........ r73811 | benjamin.peterson | 2009-07-03 09:03:14 -0500 (Fri, 03 Jul 2009) | 1 line check for sep, not pathsep when looking for a subpackage #6408 ........ r73840 | benjamin.peterson | 2009-07-04 09:52:28 -0500 (Sat, 04 Jul 2009) | 1 line don't print diffs by default; it's annoying ........ r73842 | benjamin.peterson | 2009-07-04 09:58:46 -0500 (Sat, 04 Jul 2009) | 1 line complain when not showing diffs or writing ........ r73848 | alexandre.vassalotti | 2009-07-04 23:38:19 -0500 (Sat, 04 Jul 2009) | 2 lines Fix test_refactor_stdin to handle print_output() method with 4 arguments. ........ r73849 | alexandre.vassalotti | 2009-07-04 23:43:18 -0500 (Sat, 04 Jul 2009) | 5 lines Issue 2370: Add fixer for the removal of operator.isCallable() and operator.sequenceIncludes(). Patch contributed by Jeff Balogh (and updated by me). ........ r73861 | benjamin.peterson | 2009-07-05 09:15:53 -0500 (Sun, 05 Jul 2009) | 1 line cleanup and use unicode where appropiate ........ r73957 | benjamin.peterson | 2009-07-11 15:49:56 -0500 (Sat, 11 Jul 2009) | 1 line fix calls to str() with unicode() ........ r73958 | benjamin.peterson | 2009-07-11 15:51:51 -0500 (Sat, 11 Jul 2009) | 1 line more str() -> unicode() ........ r73959 | benjamin.peterson | 2009-07-11 16:40:08 -0500 (Sat, 11 Jul 2009) | 1 line add tests for refactor_dir() ........ r73960 | benjamin.peterson | 2009-07-11 16:44:32 -0500 (Sat, 11 Jul 2009) | 1 line don't parse files just because they end with 'py' (no dot) ........ r73964 | benjamin.peterson | 2009-07-11 17:30:15 -0500 (Sat, 11 Jul 2009) | 1 line simplify ........ r73965 | benjamin.peterson | 2009-07-11 17:31:30 -0500 (Sat, 11 Jul 2009) | 1 line remove usage of get_prefix() ........ r73966 | benjamin.peterson | 2009-07-11 17:33:35 -0500 (Sat, 11 Jul 2009) | 1 line revert unintended change in 73965 ........ r73967 | benjamin.peterson | 2009-07-11 17:34:44 -0500 (Sat, 11 Jul 2009) | 1 line avoid expensive checks and assume the node did change ........ r73968 | benjamin.peterson | 2009-07-11 20:46:46 -0500 (Sat, 11 Jul 2009) | 1 line use a regular dict for the heads to avoid adding lists in the loop ........ r73969 | benjamin.peterson | 2009-07-11 20:50:43 -0500 (Sat, 11 Jul 2009) | 1 line prefix headnode functions with '_' ........ r73972 | benjamin.peterson | 2009-07-11 21:25:45 -0500 (Sat, 11 Jul 2009) | 1 line try to make the head node dict as sparse as possible ........ r73973 | benjamin.peterson | 2009-07-11 21:59:49 -0500 (Sat, 11 Jul 2009) | 1 line a better idea; add an option to *not* print diffs ........ r73974 | benjamin.peterson | 2009-07-11 22:00:29 -0500 (Sat, 11 Jul 2009) | 1 line add space ........ r73977 | benjamin.peterson | 2009-07-12 10:16:07 -0500 (Sun, 12 Jul 2009) | 1 line update get_headnode_dict tests for recent changes ........ r73981 | benjamin.peterson | 2009-07-12 12:06:39 -0500 (Sun, 12 Jul 2009) | 4 lines detect when "from __future__ import print_function" is given Deprecate the 'print_function' option and the -p flag ........ r73984 | benjamin.peterson | 2009-07-12 16:16:37 -0500 (Sun, 12 Jul 2009) | 1 line add tests for Call; thanks Joe Amenta ........ r74065 | benjamin.peterson | 2009-07-17 12:52:49 -0500 (Fri, 17 Jul 2009) | 1 line pathname2url and url2pathname are in urllib.request not urllib.parse #6496 ........ r74113 | benjamin.peterson | 2009-07-20 08:56:57 -0500 (Mon, 20 Jul 2009) | 1 line fix deprecation warnings in tests ........
2009-07-20 12:33:09 -03:00
def _type_of_literal(value):
if value[0].isalpha():
return token.NAME
elif value in grammar.opmap:
return grammar.opmap[value]
else:
return None
2008-03-19 01:43:46 -03:00
def pattern_convert(grammar, raw_node_info):
"""Converts raw node information to a Node or Leaf instance."""
type, value, context, children = raw_node_info
if children or type in grammar.number2symbol:
return pytree.Node(type, children, context=context)
else:
return pytree.Leaf(type, value, context=context)
def compile_pattern(pattern):
return PatternCompiler().compile_pattern(pattern)