From db23b8bb13863fcd88ff91bc22398f8e0312039e Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Sun, 6 Oct 2024 15:16:41 +0200 Subject: [PATCH] gh-125008: Fix `tokenize.untokenize` roundtrip for `\n{{` (#125013) --- Lib/test/test_tokenize.py | 20 +++++++++++++++++++ Lib/tokenize.py | 2 +- ...-10-05-23-53-06.gh-issue-125008.ETANpd.rst | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index de0e0b430a2..75710db7d05 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1919,6 +1919,26 @@ class TestRoundtrip(TestCase): self.check_roundtrip(r"f'\\\\N{{'") self.check_roundtrip(r"f'\\\\\\N{{'") self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") cases = [ """ if 1: diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7f418bb7a1b..4b4c3cfe169 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -200,7 +200,7 @@ class Untokenizer: characters[-2::-1] ) ) - if n_backslashes % 2 == 0: + if n_backslashes % 2 == 0 or characters[-1] != "N": characters.append(character) else: consume_until_next_bracket = True diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst new file mode 100644 index 00000000000..8971e052860 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst @@ -0,0 +1,2 @@ +Fix :func:`tokenize.untokenize` producing invalid syntax for +double braces preceded by certain escape characters.