Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-104866: Tokenize should emit NEWLINE after exiting block with comment (#104870)

This commit is contained in:
Lysandros Nikolaou 2023-05-24 19:18:17 +03:00 committed by GitHub
parent 9d457e1154
commit c90a862cdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Lib/test/test_tokenize.py
View File

@ -1057,6 +1057,23 @@ async def f():
DEDENT '' (6, 12) (6, 12)
""")
def test_newline_after_parenthesized_block_with_comment(self):
self.check_tokenize('''\
[
# A comment here
1
]
''', """\
OP '[' (1, 0) (1, 1)
NL '\\n' (1, 1) (1, 2)
COMMENT '# A comment here' (2, 4) (2, 20)
NL '\\n' (2, 20) (2, 21)
NUMBER '1' (3, 4) (3, 5)
NL '\\n' (3, 5) (3, 6)
OP ']' (4, 0) (4, 1)
NEWLINE '\\n' (4, 1) (4, 2)
""")
class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.

9
Parser/tokenizer.c
View File

@ -2007,6 +2007,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
tok->atbol = 1;
if (blankline || tok->level > 0) {
if (tok->tok_extra_tokens) {
if (tok->comment_newline) {
tok->comment_newline = 0;
}
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
@ -2015,9 +2018,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
}
if (tok->comment_newline && tok->tok_extra_tokens) {
tok->comment_newline = 0;
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
}
p_start = tok->start;
p_end = tok->cur - 1; /* Leave '\n' out of the string */