Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-105820: Fix tok_mode expression buffer in file & readline tokenizer (#105828)

This commit is contained in:
Lysandros Nikolaou 2023-06-15 18:21:24 +02:00 committed by GitHub
parent 8f10140e74
commit d382ad4915
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 36 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Lib/test/test_fstring.py
View File

@ -15,7 +15,7 @@ import decimal
import unittest
from test import support
from test.support.os_helper import temp_cwd
from test.support.script_helper import assert_python_failure
from test.support.script_helper import assert_python_failure, assert_python_ok
a_global = 'global variable'
@ -1635,5 +1635,18 @@ sdfsdfs{1+
"f'{1=}{1;}'",
])
def test_debug_in_file(self):
with temp_cwd():
script = 'script.py'
with open('script.py', 'w') as f:
f.write(f"""\
print(f'''{{
3
=}}''')""")
_, stdout, _ = assert_python_ok(script)
self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
"3\n=3")
if __name__ == '__main__':
unittest.main()

13
Lib/test/test_tokenize.py
View File

@ -558,6 +558,19 @@ def"', """\
OP '}' (1, 39) (1, 40)
FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
FSTRING_END "'" (1, 52) (1, 53)
""")
self.check_tokenize("""\
f'''{
3
=}'''""", """\
FSTRING_START "f'''" (1, 0) (1, 4)
OP '{' (1, 4) (1, 5)
NL '\\n' (1, 5) (1, 6)
NUMBER '3' (2, 0) (2, 1)
NL '\\n' (2, 1) (2, 2)
OP '=' (3, 0) (3, 1)
OP '}' (3, 1) (3, 2)
FSTRING_END "'''" (3, 2) (3, 5)
""")
def test_function(self):

1
Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst Normal file
View File

@ -0,0 +1 @@
Fix an f-string bug, where using a debug expression (the `=` sign) that appears in the last line of a file results to the debug buffer that holds the expression text being one character too small.

11
Parser/tokenizer.c
View File

@ -1039,9 +1039,6 @@ tok_readline_raw(struct tok_state *tok)
if (line == NULL) {
return 1;
}
if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
return 0;
}
if (tok->fp_interactive &&
tok_concatenate_interactive_new_line(tok, line) == -1) {
return 0;
@ -1270,6 +1267,10 @@ tok_underflow_file(struct tok_state *tok) {
tok->implicit_newline = 1;
}
if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
return 0;
}
ADVANCE_LINENO();
if (tok->decoding_state != STATE_NORMAL) {
if (tok->lineno > 2) {
@ -1314,6 +1315,10 @@ tok_underflow_readline(struct tok_state* tok) {
tok->implicit_newline = 1;
}
if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
return 0;
}
ADVANCE_LINENO();
/* The default encoding is UTF-8, so make sure we don't have any
non-UTF-8 sequences in it. */