bpo-32874: IDLE: add tests for pyparse (GH-5755)
There are no code changes other than comments and docstrings.
This commit is contained in:
Cheryl Sabella
Terry Jan Reedy
parent
ba518804bf
commit
c84cf6c03f
|
|
@ -0,0 +1,523 @@
|
|||
"""Unittest for idlelib.pyparse.py."""
|
||||
|
||||
from collections import namedtuple
|
||||
import unittest
|
||||
from idlelib import pyparse
|
||||
|
||||
|
||||
class StringTranslatePseudoMappingTest(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
whitespace_chars = ' \t\n\r'
|
||||
cls.preserve_dict = {ord(c): ord(c) for c in whitespace_chars}
|
||||
cls.default = ord('x')
|
||||
cls.mapping = pyparse.StringTranslatePseudoMapping(
|
||||
cls.preserve_dict, default_value=ord('x'))
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
del cls.preserve_dict, cls.default, cls.mapping
|
||||
|
||||
def test__init__(self):
|
||||
m = self.mapping
|
||||
self.assertEqual(m._non_defaults, self.preserve_dict)
|
||||
self.assertEqual(m._default_value, self.default)
|
||||
|
||||
def test__get_item__(self):
|
||||
self.assertEqual(self.mapping[ord('\t')], ord('\t'))
|
||||
self.assertEqual(self.mapping[ord('a')], self.default)
|
||||
|
||||
def test__len__(self):
|
||||
self.assertEqual(len(self.mapping), len(self.preserve_dict))
|
||||
|
||||
def test__iter__(self):
|
||||
count = 0
|
||||
for key, value in self.mapping.items():
|
||||
self.assertIn(key, self.preserve_dict)
|
||||
count += 1
|
||||
self.assertEqual(count, len(self.mapping))
|
||||
|
||||
def test_get(self):
|
||||
self.assertEqual(self.mapping.get(ord('\t')), ord('\t'))
|
||||
self.assertEqual(self.mapping.get('a'), self.default)
|
||||
# Default is a parameter, but it isn't used.
|
||||
self.assertEqual(self.mapping.get('a', default=500), self.default)
|
||||
|
||||
|
||||
class PyParseTest(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.parser = pyparse.Parser(indentwidth=4, tabwidth=4)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
del cls.parser
|
||||
|
||||
def test_init(self):
|
||||
self.assertEqual(self.parser.indentwidth, 4)
|
||||
self.assertEqual(self.parser.tabwidth, 4)
|
||||
|
||||
def test_set_str(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
|
||||
# Not empty and doesn't end with newline.
|
||||
with self.assertRaises(AssertionError):
|
||||
setstr('a')
|
||||
|
||||
tests = ('',
|
||||
'a\n')
|
||||
|
||||
for string in tests:
|
||||
with self.subTest(string=string):
|
||||
setstr(string)
|
||||
eq(p.str, string)
|
||||
eq(p.study_level, 0)
|
||||
|
||||
def test_find_good_parse_start(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
start = p.find_good_parse_start
|
||||
|
||||
# Split def across lines.
|
||||
setstr('"""This is a module docstring"""\n'
|
||||
'class C():\n'
|
||||
' def __init__(self, a,\n'
|
||||
' b=True):\n'
|
||||
' pass\n'
|
||||
)
|
||||
|
||||
# No value sent for is_char_in_string().
|
||||
self.assertIsNone(start())
|
||||
|
||||
# Make text look like a string. This returns pos as the start
|
||||
# position, but it's set to None.
|
||||
self.assertIsNone(start(is_char_in_string=lambda index: True))
|
||||
|
||||
# Make all text look like it's not in a string. This means that it
|
||||
# found a good start position.
|
||||
eq(start(is_char_in_string=lambda index: False), 44)
|
||||
|
||||
# If the beginning of the def line is not in a string, then it
|
||||
# returns that as the index.
|
||||
eq(start(is_char_in_string=lambda index: index > 44), 44)
|
||||
# If the beginning of the def line is in a string, then it
|
||||
# looks for a previous index.
|
||||
eq(start(is_char_in_string=lambda index: index >= 44), 33)
|
||||
# If everything before the 'def' is in a string, then returns None.
|
||||
# The non-continuation def line returns 44 (see below).
|
||||
eq(start(is_char_in_string=lambda index: index < 44), None)
|
||||
|
||||
# Code without extra line break in def line - mostly returns the same
|
||||
# values.
|
||||
setstr('"""This is a module docstring"""\n'
|
||||
'class C():\n'
|
||||
' def __init__(self, a, b=True):\n'
|
||||
' pass\n'
|
||||
)
|
||||
eq(start(is_char_in_string=lambda index: False), 44)
|
||||
eq(start(is_char_in_string=lambda index: index > 44), 44)
|
||||
eq(start(is_char_in_string=lambda index: index >= 44), 33)
|
||||
# When the def line isn't split, this returns which doesn't match the
|
||||
# split line test.
|
||||
eq(start(is_char_in_string=lambda index: index < 44), 44)
|
||||
|
||||
def test_set_lo(self):
|
||||
code = (
|
||||
'"""This is a module docstring"""\n'
|
||||
'class C():\n'
|
||||
' def __init__(self, a,\n'
|
||||
' b=True):\n'
|
||||
' pass\n'
|
||||
)
|
||||
p = self.parser
|
||||
p.set_str(code)
|
||||
|
||||
# Previous character is not a newline.
|
||||
with self.assertRaises(AssertionError):
|
||||
p.set_lo(5)
|
||||
|
||||
# A value of 0 doesn't change self.str.
|
||||
p.set_lo(0)
|
||||
self.assertEqual(p.str, code)
|
||||
|
||||
# An index that is preceded by a newline.
|
||||
p.set_lo(44)
|
||||
self.assertEqual(p.str, code[44:])
|
||||
|
||||
def test_tran(self):
|
||||
self.assertEqual('\t a([{b}])b"c\'d\n'.translate(self.parser._tran),
|
||||
'xxx(((x)))x"x\'x\n')
|
||||
|
||||
def test_study1(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
study = p._study1
|
||||
|
||||
(NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'goodlines',
|
||||
'continuation'])
|
||||
tests = (
|
||||
TestInfo('', [0], NONE),
|
||||
# Docstrings.
|
||||
TestInfo('"""This is a complete docstring."""\n', [0, 1], NONE),
|
||||
TestInfo("'''This is a complete docstring.'''\n", [0, 1], NONE),
|
||||
TestInfo('"""This is a continued docstring.\n', [0, 1], FIRST),
|
||||
TestInfo("'''This is a continued docstring.\n", [0, 1], FIRST),
|
||||
TestInfo('"""Closing quote does not match."\n', [0, 1], FIRST),
|
||||
TestInfo('"""Bracket in docstring [\n', [0, 1], FIRST),
|
||||
TestInfo("'''Incomplete two line docstring.\n\n", [0, 2], NEXT),
|
||||
# Single-quoted strings.
|
||||
TestInfo('"This is a complete string."\n', [0, 1], NONE),
|
||||
TestInfo('"This is an incomplete string.\n', [0, 1], NONE),
|
||||
TestInfo("'This is more incomplete.\n\n", [0, 1, 2], NONE),
|
||||
# Comment (backslash does not continue comments).
|
||||
TestInfo('# Comment\\\n', [0, 1], NONE),
|
||||
# Brackets.
|
||||
TestInfo('("""Complete string in bracket"""\n', [0, 1], BRACKET),
|
||||
TestInfo('("""Open string in bracket\n', [0, 1], FIRST),
|
||||
TestInfo('a = (1 + 2) - 5 *\\\n', [0, 1], BACKSLASH), # No bracket.
|
||||
TestInfo('\n def function1(self, a,\n b):\n',
|
||||
[0, 1, 3], NONE),
|
||||
TestInfo('\n def function1(self, a,\\\n', [0, 1, 2], BRACKET),
|
||||
TestInfo('\n def function1(self, a,\n', [0, 1, 2], BRACKET),
|
||||
TestInfo('())\n', [0, 1], NONE), # Extra closer.
|
||||
TestInfo(')(\n', [0, 1], BRACKET), # Extra closer.
|
||||
# For the mismatched example, it doesn't look like contination.
|
||||
TestInfo('{)(]\n', [0, 1], NONE), # Mismatched.
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string) # resets study_level
|
||||
study()
|
||||
eq(p.study_level, 1)
|
||||
eq(p.goodlines, test.goodlines)
|
||||
eq(p.continuation, test.continuation)
|
||||
|
||||
# Called again, just returns without reprocessing.
|
||||
self.assertIsNone(study())
|
||||
|
||||
def test_get_continuation_type(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
gettype = p.get_continuation_type
|
||||
|
||||
(NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'continuation'])
|
||||
tests = (
|
||||
TestInfo('', NONE),
|
||||
TestInfo('"""This is a continuation docstring.\n', FIRST),
|
||||
TestInfo("'''This is a multiline-continued docstring.\n\n", NEXT),
|
||||
TestInfo('a = (1 + 2) - 5 *\\\n', BACKSLASH),
|
||||
TestInfo('\n def function1(self, a,\\\n', BRACKET)
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(gettype(), test.continuation)
|
||||
|
||||
def test_study2(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
study = p._study2
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'start', 'end', 'lastch',
|
||||
'openbracket', 'bracketing'])
|
||||
tests = (
|
||||
TestInfo('', 0, 0, '', None, ((0, 0),)),
|
||||
TestInfo("'''This is a multiline continutation docstring.\n\n",
|
||||
0, 49, "'", None, ((0, 0), (0, 1), (49, 0))),
|
||||
TestInfo(' # Comment\\\n',
|
||||
0, 12, '', None, ((0, 0), (1, 1), (12, 0))),
|
||||
# A comment without a space is a special case
|
||||
TestInfo(' #Comment\\\n',
|
||||
0, 0, '', None, ((0, 0),)),
|
||||
# Backslash continuation.
|
||||
TestInfo('a = (1 + 2) - 5 *\\\n',
|
||||
0, 19, '*', None, ((0, 0), (4, 1), (11, 0))),
|
||||
# Bracket continuation with close.
|
||||
TestInfo('\n def function1(self, a,\n b):\n',
|
||||
1, 48, ':', None, ((1, 0), (17, 1), (46, 0))),
|
||||
# Bracket continuation with unneeded backslash.
|
||||
TestInfo('\n def function1(self, a,\\\n',
|
||||
1, 28, ',', 17, ((1, 0), (17, 1))),
|
||||
# Bracket continuation.
|
||||
TestInfo('\n def function1(self, a,\n',
|
||||
1, 27, ',', 17, ((1, 0), (17, 1))),
|
||||
# Bracket continuation with comment at end of line with text.
|
||||
TestInfo('\n def function1(self, a, # End of line comment.\n',
|
||||
1, 51, ',', 17, ((1, 0), (17, 1), (28, 2), (51, 1))),
|
||||
# Multi-line statement with comment line in between code lines.
|
||||
TestInfo(' a = ["first item",\n # Comment line\n "next item",\n',
|
||||
0, 55, ',', 6, ((0, 0), (6, 1), (7, 2), (19, 1),
|
||||
(23, 2), (38, 1), (42, 2), (53, 1))),
|
||||
TestInfo('())\n',
|
||||
0, 4, ')', None, ((0, 0), (0, 1), (2, 0), (3, 0))),
|
||||
TestInfo(')(\n', 0, 3, '(', 1, ((0, 0), (1, 0), (1, 1))),
|
||||
# Wrong closers still decrement stack level.
|
||||
TestInfo('{)(]\n',
|
||||
0, 5, ']', None, ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
|
||||
# Character after backslash.
|
||||
TestInfo(':\\a\n', 0, 4, '\\a', None, ((0, 0),)),
|
||||
TestInfo('\n', 0, 0, '', None, ((0, 0),)),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
# There is a bug where this is carried forward from last item.
|
||||
p.lastopenbracketpos = None
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
study()
|
||||
eq(p.study_level, 2)
|
||||
eq(p.stmt_start, test.start)
|
||||
eq(p.stmt_end, test.end)
|
||||
eq(p.lastch, test.lastch)
|
||||
eq(p.lastopenbracketpos, test.openbracket)
|
||||
eq(p.stmt_bracketing, test.bracketing)
|
||||
|
||||
# Called again, just returns without reprocessing.
|
||||
self.assertIsNone(study())
|
||||
|
||||
def test_get_num_lines_in_stmt(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
getlines = p.get_num_lines_in_stmt
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'lines'])
|
||||
tests = (
|
||||
TestInfo('[x for x in a]\n', 1), # Closed on one line.
|
||||
TestInfo('[x\nfor x in a\n', 2), # Not closed.
|
||||
TestInfo('[x\\\nfor x in a\\\n', 2), # "", uneeded backslashes.
|
||||
TestInfo('[x\nfor x in a\n]\n', 3), # Closed on multi-line.
|
||||
TestInfo('\n"""Docstring comment L1"""\nL2\nL3\nL4\n', 1),
|
||||
TestInfo('\n"""Docstring comment L1\nL2"""\nL3\nL4\n', 1),
|
||||
TestInfo('\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n', 4),
|
||||
TestInfo('\n\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n"""\n', 5)
|
||||
)
|
||||
|
||||
# Blank string doesn't have enough elements in goodlines.
|
||||
setstr('')
|
||||
with self.assertRaises(IndexError):
|
||||
getlines()
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(getlines(), test.lines)
|
||||
|
||||
def test_compute_bracket_indent(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
indent = p.compute_bracket_indent
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'spaces'])
|
||||
tests = (
|
||||
TestInfo('def function1(self, a,\n', 14),
|
||||
# Characters after bracket.
|
||||
TestInfo('\n def function1(self, a,\n', 18),
|
||||
TestInfo('\n\tdef function1(self, a,\n', 18),
|
||||
# No characters after bracket.
|
||||
TestInfo('\n def function1(\n', 8),
|
||||
TestInfo('\n\tdef function1(\n', 8),
|
||||
TestInfo('\n def function1( \n', 8), # Ignore extra spaces.
|
||||
TestInfo('[\n"first item",\n # Comment line\n "next item",\n', 0),
|
||||
TestInfo('[\n "first item",\n # Comment line\n "next item",\n', 2),
|
||||
TestInfo('["first item",\n # Comment line\n "next item",\n', 1),
|
||||
TestInfo('(\n', 4),
|
||||
TestInfo('(a\n', 1),
|
||||
)
|
||||
|
||||
# Must be C_BRACKET continuation type.
|
||||
setstr('def function1(self, a, b):\n')
|
||||
with self.assertRaises(AssertionError):
|
||||
indent()
|
||||
|
||||
for test in tests:
|
||||
setstr(test.string)
|
||||
eq(indent(), test.spaces)
|
||||
|
||||
def test_compute_backslash_indent(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
indent = p.compute_backslash_indent
|
||||
|
||||
# Must be C_BACKSLASH continuation type.
|
||||
errors = (('def function1(self, a, b\\\n'), # Bracket.
|
||||
(' """ (\\\n'), # Docstring.
|
||||
('a = #\\\n'), # Inline comment.
|
||||
)
|
||||
for string in errors:
|
||||
with self.subTest(string=string):
|
||||
setstr(string)
|
||||
with self.assertRaises(AssertionError):
|
||||
indent()
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ('string', 'spaces'))
|
||||
tests = (TestInfo('a = (1 + 2) - 5 *\\\n', 4),
|
||||
TestInfo('a = 1 + 2 - 5 *\\\n', 4),
|
||||
TestInfo(' a = 1 + 2 - 5 *\\\n', 8),
|
||||
TestInfo(' a = "spam"\\\n', 6),
|
||||
TestInfo(' a = \\\n"a"\\\n', 4),
|
||||
TestInfo(' a = #\\\n"a"\\\n', 5),
|
||||
TestInfo('a == \\\n', 2),
|
||||
TestInfo('a != \\\n', 2),
|
||||
# Difference between containing = and those not.
|
||||
TestInfo('\\\n', 2),
|
||||
TestInfo(' \\\n', 6),
|
||||
TestInfo('\t\\\n', 6),
|
||||
TestInfo('a\\\n', 3),
|
||||
TestInfo('{}\\\n', 4),
|
||||
TestInfo('(1 + 2) - 5 *\\\n', 3),
|
||||
)
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(indent(), test.spaces)
|
||||
|
||||
def test_get_base_indent_string(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
baseindent = p.get_base_indent_string
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'indent'])
|
||||
tests = (TestInfo('', ''),
|
||||
TestInfo('def a():\n', ''),
|
||||
TestInfo('\tdef a():\n', '\t'),
|
||||
TestInfo(' def a():\n', ' '),
|
||||
TestInfo(' def a(\n', ' '),
|
||||
TestInfo('\t\n def a(\n', ' '),
|
||||
TestInfo('\t\n # Comment.\n', ' '),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(baseindent(), test.indent)
|
||||
|
||||
def test_is_block_opener(self):
|
||||
yes = self.assertTrue
|
||||
no = self.assertFalse
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
opener = p.is_block_opener
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
|
||||
tests = (
|
||||
TestInfo('def a():\n', yes),
|
||||
TestInfo('\n def function1(self, a,\n b):\n', yes),
|
||||
TestInfo(':\n', yes),
|
||||
TestInfo('a:\n', yes),
|
||||
TestInfo('):\n', yes),
|
||||
TestInfo('(:\n', yes),
|
||||
TestInfo('":\n', no),
|
||||
TestInfo('\n def function1(self, a,\n', no),
|
||||
TestInfo('def function1(self, a):\n pass\n', no),
|
||||
TestInfo('# A comment:\n', no),
|
||||
TestInfo('"""A docstring:\n', no),
|
||||
TestInfo('"""A docstring:\n', no),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
test.assert_(opener())
|
||||
|
||||
def test_is_block_closer(self):
|
||||
yes = self.assertTrue
|
||||
no = self.assertFalse
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
closer = p.is_block_closer
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
|
||||
tests = (
|
||||
TestInfo('return\n', yes),
|
||||
TestInfo('\tbreak\n', yes),
|
||||
TestInfo(' continue\n', yes),
|
||||
TestInfo(' raise\n', yes),
|
||||
TestInfo('pass \n', yes),
|
||||
TestInfo('pass\t\n', yes),
|
||||
TestInfo('return #\n', yes),
|
||||
TestInfo('raised\n', no),
|
||||
TestInfo('returning\n', no),
|
||||
TestInfo('# return\n', no),
|
||||
TestInfo('"""break\n', no),
|
||||
TestInfo('"continue\n', no),
|
||||
TestInfo('def function1(self, a):\n pass\n', yes),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
test.assert_(closer())
|
||||
|
||||
def test_get_last_open_bracket_pos(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
openbracket = p.get_last_open_bracket_pos
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'position'])
|
||||
tests = (
|
||||
TestInfo('', None),
|
||||
TestInfo('a\n', None),
|
||||
TestInfo('# (\n', None),
|
||||
TestInfo('""" (\n', None),
|
||||
TestInfo('a = (1 + 2) - 5 *\\\n', None),
|
||||
TestInfo('\n def function1(self, a,\n', 17),
|
||||
TestInfo('\n def function1(self, a, # End of line comment.\n', 17),
|
||||
TestInfo('{)(]\n', None),
|
||||
TestInfo('(((((((((()))))))\n', 2),
|
||||
TestInfo('(((((((((())\n)))\n))\n', 2),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
# There is a bug where the value is carried forward from last item.
|
||||
p.lastopenbracketpos = None
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(openbracket(), test.position)
|
||||
|
||||
def test_get_last_stmt_bracketing(self):
|
||||
eq = self.assertEqual
|
||||
p = self.parser
|
||||
setstr = p.set_str
|
||||
bracketing = p.get_last_stmt_bracketing
|
||||
|
||||
TestInfo = namedtuple('TestInfo', ['string', 'bracket'])
|
||||
tests = (
|
||||
TestInfo('', ((0, 0),)),
|
||||
TestInfo('a\n', ((0, 0),)),
|
||||
TestInfo('()()\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
|
||||
TestInfo('(\n)()\n', ((0, 0), (0, 1), (3, 0), (3, 1), (5, 0))),
|
||||
TestInfo('()\n()\n', ((3, 0), (3, 1), (5, 0))),
|
||||
TestInfo('()(\n)\n', ((0, 0), (0, 1), (2, 0), (2, 1), (5, 0))),
|
||||
TestInfo('(())\n', ((0, 0), (0, 1), (1, 2), (3, 1), (4, 0))),
|
||||
TestInfo('(\n())\n', ((0, 0), (0, 1), (2, 2), (4, 1), (5, 0))),
|
||||
# Same as matched test.
|
||||
TestInfo('{)(]\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
|
||||
TestInfo('(((())\n',
|
||||
((0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (5, 3), (6, 2))),
|
||||
)
|
||||
|
||||
for test in tests:
|
||||
with self.subTest(string=test.string):
|
||||
setstr(test.string)
|
||||
eq(bracketing(), test.bracket)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
|
|
@ -1,8 +1,20 @@
|
|||
"""Define partial Python code Parser used by editor and hyperparser.
|
||||
|
||||
Instances of StringTranslatePseudoMapping are used with str.translate.
|
||||
|
||||
The following bound search and match functions are defined:
|
||||
_synchre - start of popular statement;
|
||||
_junkre - whitespace or comment line;
|
||||
_match_stringre: string, possibly without closer;
|
||||
_itemre - line that may have bracket structure start;
|
||||
_closere - line that must be followed by dedent.
|
||||
_chew_ordinaryre - non-special characters.
|
||||
"""
|
||||
from collections.abc import Mapping
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Reason last stmt is continued (or C_NONE if it's not).
|
||||
# Reason last statement is continued (or C_NONE if it's not).
|
||||
(C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,
|
||||
C_STRING_NEXT_LINES, C_BRACKET) = range(5)
|
||||
|
||||
|
|
@ -10,7 +22,7 @@ if 0: # for throwaway debugging output
|
|||
def dump(*stuff):
|
||||
sys.__stdout__.write(" ".join(map(str, stuff)) + "\n")
|
||||
|
||||
# Find what looks like the start of a popular stmt.
|
||||
# Find what looks like the start of a popular statement.
|
||||
|
||||
_synchre = re.compile(r"""
|
||||
^
|
||||
|
|
@ -70,7 +82,7 @@ _itemre = re.compile(r"""
|
|||
[^\s#\\] # if we match, m.end()-1 is the interesting char
|
||||
""", re.VERBOSE).match
|
||||
|
||||
# Match start of stmts that should be followed by a dedent.
|
||||
# Match start of statements that should be followed by a dedent.
|
||||
|
||||
_closere = re.compile(r"""
|
||||
\s*
|
||||
|
|
@ -146,19 +158,20 @@ class Parser:
|
|||
self.str = s
|
||||
self.study_level = 0
|
||||
|
||||
# Return index of a good place to begin parsing, as close to the
|
||||
# end of the string as possible. This will be the start of some
|
||||
# popular stmt like "if" or "def". Return None if none found:
|
||||
# the caller should pass more prior context then, if possible, or
|
||||
# if not (the entire program text up until the point of interest
|
||||
# has already been tried) pass 0 to set_lo.
|
||||
#
|
||||
# This will be reliable iff given a reliable is_char_in_string
|
||||
# function, meaning that when it says "no", it's absolutely
|
||||
# guaranteed that the char is not in a string.
|
||||
|
||||
def find_good_parse_start(self, is_char_in_string=None,
|
||||
_synchre=_synchre):
|
||||
"""
|
||||
Return index of a good place to begin parsing, as close to the
|
||||
end of the string as possible. This will be the start of some
|
||||
popular stmt like "if" or "def". Return None if none found:
|
||||
the caller should pass more prior context then, if possible, or
|
||||
if not (the entire program text up until the point of interest
|
||||
has already been tried) pass 0 to set_lo().
|
||||
|
||||
This will be reliable iff given a reliable is_char_in_string()
|
||||
function, meaning that when it says "no", it's absolutely
|
||||
guaranteed that the char is not in a string.
|
||||
"""
|
||||
str, pos = self.str, None
|
||||
|
||||
if not is_char_in_string:
|
||||
|
|
@ -173,7 +186,7 @@ class Parser:
|
|||
i = str.rfind(":\n", 0, limit)
|
||||
if i < 0:
|
||||
break
|
||||
i = str.rfind('\n', 0, i) + 1 # start of colon line
|
||||
i = str.rfind('\n', 0, i) + 1 # start of colon line (-1+1=0)
|
||||
m = _synchre(str, i, limit)
|
||||
if m and not is_char_in_string(m.start()):
|
||||
pos = m.start()
|
||||
|
|
@ -206,10 +219,11 @@ class Parser:
|
|||
break
|
||||
return pos
|
||||
|
||||
# Throw away the start of the string. Intended to be called with
|
||||
# find_good_parse_start's result.
|
||||
|
||||
def set_lo(self, lo):
|
||||
""" Throw away the start of the string.
|
||||
|
||||
Intended to be called with the result of find_good_parse_start().
|
||||
"""
|
||||
assert lo == 0 or self.str[lo-1] == '\n'
|
||||
if lo > 0:
|
||||
self.str = self.str[lo:]
|
||||
|
|
@ -224,11 +238,13 @@ class Parser:
|
|||
_tran.update((ord(c), ord(c)) for c in "\"'\\\n#")
|
||||
_tran = StringTranslatePseudoMapping(_tran, default_value=ord('x'))
|
||||
|
||||
# As quickly as humanly possible <wink>, find the line numbers (0-
|
||||
# based) of the non-continuation lines.
|
||||
# Creates self.{goodlines, continuation}.
|
||||
|
||||
def _study1(self):
|
||||
"""Find the line numbers of non-continuation lines.
|
||||
|
||||
As quickly as humanly possible <wink>, find the line numbers (0-
|
||||
based) of the non-continuation lines.
|
||||
Creates self.{goodlines, continuation}.
|
||||
"""
|
||||
if self.study_level >= 1:
|
||||
return
|
||||
self.study_level = 1
|
||||
|
|
@ -244,8 +260,8 @@ class Parser:
|
|||
str = str.replace('xx', 'x')
|
||||
str = str.replace('xx', 'x')
|
||||
str = str.replace('\nx', '\n')
|
||||
# note that replacing x\n with \n would be incorrect, because
|
||||
# x may be preceded by a backslash
|
||||
# Replacing x\n with \n would be incorrect because
|
||||
# x may be preceded by a backslash.
|
||||
|
||||
# March over the squashed version of the program, accumulating
|
||||
# the line numbers of non-continued stmts, and determining
|
||||
|
|
@ -360,24 +376,25 @@ class Parser:
|
|||
self._study1()
|
||||
return self.continuation
|
||||
|
||||
# study1 was sufficient to determine the continuation status,
|
||||
# but doing more requires looking at every character. study2
|
||||
# does this for the last interesting statement in the block.
|
||||
# Creates:
|
||||
# self.stmt_start, stmt_end
|
||||
# slice indices of last interesting stmt
|
||||
# self.stmt_bracketing
|
||||
# the bracketing structure of the last interesting stmt;
|
||||
# for example, for the statement "say(boo) or die", stmt_bracketing
|
||||
# will be [(0, 0), (3, 1), (8, 0)]. Strings and comments are
|
||||
# treated as brackets, for the matter.
|
||||
# self.lastch
|
||||
# last non-whitespace character before optional trailing
|
||||
# comment
|
||||
# self.lastopenbracketpos
|
||||
# if continuation is C_BRACKET, index of last open bracket
|
||||
|
||||
def _study2(self):
|
||||
"""
|
||||
study1 was sufficient to determine the continuation status,
|
||||
but doing more requires looking at every character. study2
|
||||
does this for the last interesting statement in the block.
|
||||
Creates:
|
||||
self.stmt_start, stmt_end
|
||||
slice indices of last interesting stmt
|
||||
self.stmt_bracketing
|
||||
the bracketing structure of the last interesting stmt; for
|
||||
example, for the statement "say(boo) or die",
|
||||
stmt_bracketing will be ((0, 0), (0, 1), (2, 0), (2, 1),
|
||||
(4, 0)). Strings and comments are treated as brackets, for
|
||||
the matter.
|
||||
self.lastch
|
||||
last interesting character before optional trailing comment
|
||||
self.lastopenbracketpos
|
||||
if continuation is C_BRACKET, index of last open bracket
|
||||
"""
|
||||
if self.study_level >= 2:
|
||||
return
|
||||
self._study1()
|
||||
|
|
@ -385,11 +402,11 @@ class Parser:
|
|||
|
||||
# Set p and q to slice indices of last interesting stmt.
|
||||
str, goodlines = self.str, self.goodlines
|
||||
i = len(goodlines) - 1
|
||||
p = len(str) # index of newest line
|
||||
i = len(goodlines) - 1 # Index of newest line.
|
||||
p = len(str) # End of goodlines[i]
|
||||
while i:
|
||||
assert p
|
||||
# p is the index of the stmt at line number goodlines[i].
|
||||
# Make p be the index of the stmt at line number goodlines[i].
|
||||
# Move p back to the stmt at line number goodlines[i-1].
|
||||
q = p
|
||||
for nothing in range(goodlines[i-1], goodlines[i]):
|
||||
|
|
@ -483,10 +500,11 @@ class Parser:
|
|||
self.lastopenbracketpos = stack[-1]
|
||||
self.stmt_bracketing = tuple(bracketing)
|
||||
|
||||
# Assuming continuation is C_BRACKET, return the number
|
||||
# of spaces the next line should be indented.
|
||||
|
||||
def compute_bracket_indent(self):
|
||||
"""Return number of spaces the next line should be indented.
|
||||
|
||||
Line continuation must be C_BRACKET.
|
||||
"""
|
||||
self._study2()
|
||||
assert self.continuation == C_BRACKET
|
||||
j = self.lastopenbracketpos
|
||||
|
|
@ -513,20 +531,22 @@ class Parser:
|
|||
extra = self.indentwidth
|
||||
return len(str[i:j].expandtabs(self.tabwidth)) + extra
|
||||
|
||||
# Return number of physical lines in last stmt (whether or not
|
||||
# it's an interesting stmt! this is intended to be called when
|
||||
# continuation is C_BACKSLASH).
|
||||
|
||||
def get_num_lines_in_stmt(self):
|
||||
"""Return number of physical lines in last stmt.
|
||||
|
||||
The statement doesn't have to be an interesting statement. This is
|
||||
intended to be called when continuation is C_BACKSLASH.
|
||||
"""
|
||||
self._study1()
|
||||
goodlines = self.goodlines
|
||||
return goodlines[-1] - goodlines[-2]
|
||||
|
||||
# Assuming continuation is C_BACKSLASH, return the number of spaces
|
||||
# the next line should be indented. Also assuming the new line is
|
||||
# the first one following the initial line of the stmt.
|
||||
|
||||
def compute_backslash_indent(self):
|
||||
"""Return number of spaces the next line should be indented.
|
||||
|
||||
Line continuation must be C_BACKSLASH. Also assume that the new
|
||||
line is the first one following the initial line of the stmt.
|
||||
"""
|
||||
self._study2()
|
||||
assert self.continuation == C_BACKSLASH
|
||||
str = self.str
|
||||
|
|
@ -551,6 +571,8 @@ class Parser:
|
|||
elif ch == '"' or ch == "'":
|
||||
i = _match_stringre(str, i, endpos).end()
|
||||
elif ch == '#':
|
||||
# This line is unreachable because the # makes a comment of
|
||||
# everything after it.
|
||||
break
|
||||
elif level == 0 and ch == '=' and \
|
||||
(i == 0 or str[i-1] not in "=<>!") and \
|
||||
|
|
@ -576,10 +598,10 @@ class Parser:
|
|||
return len(str[self.stmt_start:i].expandtabs(\
|
||||
self.tabwidth)) + 1
|
||||
|
||||
# Return the leading whitespace on the initial line of the last
|
||||
# interesting stmt.
|
||||
|
||||
def get_base_indent_string(self):
|
||||
"""Return the leading whitespace on the initial line of the last
|
||||
interesting stmt.
|
||||
"""
|
||||
self._study2()
|
||||
i, n = self.stmt_start, self.stmt_end
|
||||
j = i
|
||||
|
|
@ -588,30 +610,37 @@ class Parser:
|
|||
j = j + 1
|
||||
return str[i:j]
|
||||
|
||||
# Did the last interesting stmt open a block?
|
||||
|
||||
def is_block_opener(self):
|
||||
"Return True if the last interesting statemtent opens a block."
|
||||
self._study2()
|
||||
return self.lastch == ':'
|
||||
|
||||
# Did the last interesting stmt close a block?
|
||||
|
||||
def is_block_closer(self):
|
||||
"Return True if the last interesting statement closes a block."
|
||||
self._study2()
|
||||
return _closere(self.str, self.stmt_start) is not None
|
||||
|
||||
# index of last open bracket ({[, or None if none
|
||||
# XXX - is this used?
|
||||
lastopenbracketpos = None
|
||||
|
||||
def get_last_open_bracket_pos(self):
|
||||
"Return index of last open bracket or None."
|
||||
self._study2()
|
||||
return self.lastopenbracketpos
|
||||
|
||||
# the structure of the bracketing of the last interesting statement,
|
||||
# in the format defined in _study2, or None if the text didn't contain
|
||||
# anything
|
||||
# XXX - is this used?
|
||||
stmt_bracketing = None
|
||||
|
||||
def get_last_stmt_bracketing(self):
|
||||
"""Return a tuple of the structure of the bracketing of the last
|
||||
interesting statement.
|
||||
|
||||
Tuple is in the format defined in _study2().
|
||||
"""
|
||||
self._study2()
|
||||
return self.stmt_bracketing
|
||||
|
||||
|
||||
if __name__ == '__main__': #pragma: nocover
|
||||
import unittest
|
||||
unittest.main('idlelib.idle_test.test_pyparse', verbosity=2)
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Add tests for pyparse.
|
||||
Loading…
Reference in New Issue