[Python-checkins] bpo-45811: Improve error message when source code contains invisible control characters (GH-29654)
pablogsal
webhook-mailer at python.org
Sat Nov 20 13:28:39 EST 2021
https://github.com/python/cpython/commit/81f4e116ef7d30ef6e2041c2d6cf29af511a3a02
commit: 81f4e116ef7d30ef6e2041c2d6cf29af511a3a02
branch: main
author: Pablo Galindo Salgado <Pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-11-20T18:28:28Z
summary:
bpo-45811: Improve error message when source code contains invisible control characters (GH-29654)
files:
A Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst
M Lib/test/test_syntax.py
M Parser/tokenizer.c
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index 28414ba59493b..fc3c62954a29b 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -1566,6 +1566,9 @@ def test_error_parenthesis(self):
for paren in ")]}":
self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'")
+ def test_invisible_characters(self):
+ self._check_error('print\x17("Hello")', "invalid non-printable character")
+
def test_match_call_does_not_raise_syntax_error(self):
code = """
def match(x):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst
new file mode 100644
index 0000000000000..4b31414408150
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-20-02-25-06.bpo-45811.B-1Gsr.rst
@@ -0,0 +1,2 @@
+Improve the tokenizer errors when encountering invisible control characters
+in the parser. Patch by Pablo Galindo
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f281c423d0e0c..69d2c08b43926 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -2045,6 +2045,12 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
break;
}
+ if (!Py_UNICODE_ISPRINTABLE(c)) {
+ char hex[9];
+ (void)PyOS_snprintf(hex, sizeof(hex), "%04X", c);
+ return syntaxerror(tok, "invalid non-printable character U+%s", hex);
+ }
+
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;
More information about the Python-checkins
mailing list