From 81f4e116ef7d30ef6e2041c2d6cf29af511a3a02 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 20 Nov 2021 18:28:28 +0000 Subject: [PATCH] bpo-45811: Improve error message when source code contains invisible control characters (GH-29654) --- Lib/test/test_syntax.py | 3 +++ .../2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst | 2 ++ Parser/tokenizer.c | 6 ++++++ 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 28414ba5949..fc3c62954a2 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1566,6 +1566,9 @@ def func2(): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + def test_invisible_characters(self): + self._check_error('print\x17("Hello")', "invalid non-printable character") + def test_match_call_does_not_raise_syntax_error(self): code = """ def match(x): diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst new file mode 100644 index 00000000000..4b314144081 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst @@ -0,0 +1,2 @@ +Improve the tokenizer errors when encountering invisible control characters +in the parser. Patch by Pablo Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f281c423d0e..69d2c08b439 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2045,6 +2045,12 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) break; } + if (!Py_UNICODE_ISPRINTABLE(c)) { + char hex[9]; + (void)PyOS_snprintf(hex, sizeof(hex), "%04X", c); + return syntaxerror(tok, "invalid non-printable character U+%s", hex); + } + /* Punctuation character */ *p_start = tok->start; *p_end = tok->cur;