Issue 24226: Fix parsing of many sequential one-line 'def' statements.
This commit is contained in:
parent
a2c145c2f3
commit
8085b80c18
|
@ -1,4 +1,5 @@
|
|||
import contextlib
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
|
@ -87,6 +88,28 @@ class AsyncBadSyntaxTest(unittest.TestCase):
|
|||
import test.badsyntax_async9
|
||||
|
||||
|
||||
class TokenizerRegrTest(unittest.TestCase):
|
||||
|
||||
def test_oneline_defs(self):
|
||||
buf = []
|
||||
for i in range(500):
|
||||
buf.append('def i{i}(): return {i}'.format(i=i))
|
||||
buf = '\n'.join(buf)
|
||||
|
||||
# Test that 500 consequent, one-line defs is OK
|
||||
ns = {}
|
||||
exec(buf, ns, ns)
|
||||
self.assertEqual(ns['i499'](), 499)
|
||||
|
||||
# Test that 500 consequent, one-line defs *and*
|
||||
# one 'async def' following them is OK
|
||||
buf += '\nasync def foo():\n return'
|
||||
ns = {}
|
||||
exec(buf, ns, ns)
|
||||
self.assertEqual(ns['i499'](), 499)
|
||||
self.assertTrue(inspect.iscoroutinefunction(ns['foo']))
|
||||
|
||||
|
||||
class CoroutineTest(unittest.TestCase):
|
||||
|
||||
def test_gen_1(self):
|
||||
|
|
|
@ -1289,6 +1289,17 @@ class TestTokenize(TestCase):
|
|||
|
||||
self.assertTrue(encoding_used, encoding)
|
||||
|
||||
def test_oneline_defs(self):
|
||||
buf = []
|
||||
for i in range(500):
|
||||
buf.append('def i{i}(): return {i}'.format(i=i))
|
||||
buf.append('OK')
|
||||
buf = '\n'.join(buf)
|
||||
|
||||
# Test that 500 consequent, one-line defs is OK
|
||||
toks = list(tokenize(BytesIO(buf.encode('utf-8')).readline))
|
||||
self.assertEqual(toks[-2].string, 'OK') # [-1] is always ENDMARKER
|
||||
|
||||
def assertExactTypeEqual(self, opstr, *optypes):
|
||||
tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline))
|
||||
num_optypes = len(optypes)
|
||||
|
|
|
@ -1501,17 +1501,20 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
|
|||
|
||||
tok_len = tok->cur - tok->start;
|
||||
if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
|
||||
|
||||
if (tok->def + 1 >= MAXINDENT) {
|
||||
tok->done = E_TOODEEP;
|
||||
tok->cur = tok->inp;
|
||||
return ERRORTOKEN;
|
||||
}
|
||||
|
||||
if (tok->def && tok->deftypestack[tok->def] == 3) {
|
||||
tok->deftypestack[tok->def] = 2;
|
||||
}
|
||||
else {
|
||||
else if (tok->defstack[tok->def] < tok->indent) {
|
||||
/* We advance defs stack only when we see "def" *and*
|
||||
the indentation level was increased relative to the
|
||||
previous "def". */
|
||||
|
||||
if (tok->def + 1 >= MAXINDENT) {
|
||||
tok->done = E_TOODEEP;
|
||||
tok->cur = tok->inp;
|
||||
return ERRORTOKEN;
|
||||
}
|
||||
|
||||
tok->def++;
|
||||
tok->defstack[tok->def] = tok->indent;
|
||||
tok->deftypestack[tok->def] = 1;
|
||||
|
@ -1528,6 +1531,12 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
|
|||
ahead_tok.cur - ahead_tok.start == 3 &&
|
||||
memcmp(ahead_tok.start, "def", 3) == 0) {
|
||||
|
||||
if (tok->def + 1 >= MAXINDENT) {
|
||||
tok->done = E_TOODEEP;
|
||||
tok->cur = tok->inp;
|
||||
return ERRORTOKEN;
|
||||
}
|
||||
|
||||
tok->def++;
|
||||
tok->defstack[tok->def] = tok->indent;
|
||||
tok->deftypestack[tok->def] = 3;
|
||||
|
|
Loading…
Reference in New Issue