Issue 24226: Fix parsing of many sequential one-line 'def' statements.

This commit is contained in:
Yury Selivanov 2015-05-18 12:50:52 -04:00
parent a2c145c2f3
commit 8085b80c18
3 changed files with 51 additions and 8 deletions

View File

@ -1,4 +1,5 @@
import contextlib
import inspect
import sys
import types
import unittest
@ -87,6 +88,28 @@ class AsyncBadSyntaxTest(unittest.TestCase):
import test.badsyntax_async9
class TokenizerRegrTest(unittest.TestCase):
def test_oneline_defs(self):
buf = []
for i in range(500):
buf.append('def i{i}(): return {i}'.format(i=i))
buf = '\n'.join(buf)
# Test that 500 consequent, one-line defs is OK
ns = {}
exec(buf, ns, ns)
self.assertEqual(ns['i499'](), 499)
# Test that 500 consequent, one-line defs *and*
# one 'async def' following them is OK
buf += '\nasync def foo():\n return'
ns = {}
exec(buf, ns, ns)
self.assertEqual(ns['i499'](), 499)
self.assertTrue(inspect.iscoroutinefunction(ns['foo']))
class CoroutineTest(unittest.TestCase):
def test_gen_1(self):

View File

@ -1289,6 +1289,17 @@ class TestTokenize(TestCase):
self.assertTrue(encoding_used, encoding)
def test_oneline_defs(self):
buf = []
for i in range(500):
buf.append('def i{i}(): return {i}'.format(i=i))
buf.append('OK')
buf = '\n'.join(buf)
# Test that 500 consequent, one-line defs is OK
toks = list(tokenize(BytesIO(buf.encode('utf-8')).readline))
self.assertEqual(toks[-2].string, 'OK') # [-1] is always ENDMARKER
def assertExactTypeEqual(self, opstr, *optypes):
tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline))
num_optypes = len(optypes)

View File

@ -1501,17 +1501,20 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
tok_len = tok->cur - tok->start;
if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
if (tok->def + 1 >= MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
return ERRORTOKEN;
}
if (tok->def && tok->deftypestack[tok->def] == 3) {
tok->deftypestack[tok->def] = 2;
}
else {
else if (tok->defstack[tok->def] < tok->indent) {
/* We advance defs stack only when we see "def" *and*
the indentation level was increased relative to the
previous "def". */
if (tok->def + 1 >= MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
return ERRORTOKEN;
}
tok->def++;
tok->defstack[tok->def] = tok->indent;
tok->deftypestack[tok->def] = 1;
@ -1528,6 +1531,12 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
ahead_tok.cur - ahead_tok.start == 3 &&
memcmp(ahead_tok.start, "def", 3) == 0) {
if (tok->def + 1 >= MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
return ERRORTOKEN;
}
tok->def++;
tok->defstack[tok->def] = tok->indent;
tok->deftypestack[tok->def] = 3;