[Python-checkins] bpo-44396: Update multi-line-start location when reallocating tokenizer buffers (GH-26676) (GH-26695)
pablogsal
webhook-mailer at python.org
Sat Jun 12 16:27:11 EDT 2021
https://github.com/python/cpython/commit/d03f342a8389f1ea9100efb0d1a205601e607254
commit: d03f342a8389f1ea9100efb0d1a205601e607254
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-06-12T21:27:02+01:00
summary:
bpo-44396: Update multi-line-start location when reallocating tokenizer buffers (GH-26676) (GH-26695)
Automerge-Triggered-By: GH:pablogsal
(cherry picked from commit a342cc5891dbd8a08d40e9444f2e2c9e93258721)
files:
A Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst
M Lib/test/test_eof.py
M Parser/tokenizer.c
diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
index 2d3b4ae4e591ef..abcbf046e2cc22 100644
--- a/Lib/test/test_eof.py
+++ b/Lib/test/test_eof.py
@@ -29,6 +29,13 @@ def test_EOFS(self):
else:
raise support.TestFailed
+ def test_EOFS_with_file(self):
+ expect = ("(<string>, line 1)")
+ with os_helper.temp_dir() as temp_dir:
+ file_name = script_helper.make_script(temp_dir, 'foo', """'''this is \na \ntest""")
+ rc, out, err = script_helper.assert_python_failure(file_name)
+ self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err)
+
def test_eof_with_line_continuation(self):
expect = "unexpected EOF while parsing (<string>, line 1)"
try:
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst
new file mode 100644
index 00000000000000..be72a7111dc8ae
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-11-18-17-42.bpo-44396.Z9EKim.rst
@@ -0,0 +1,2 @@
+Fix a possible crash in the tokenizer when raising syntax errors for
+unclosed strings. Patch by Pablo Galindo.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 6002f3e05a890c..be9b13ebabb8e3 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -372,6 +372,8 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
if (newsize > tok->end - tok->buf) {
char *newbuf = tok->buf;
Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
+ Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
+ Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
newbuf = (char *)PyMem_Realloc(newbuf, newsize);
if (newbuf == NULL) {
tok->done = E_NOMEM;
@@ -382,6 +384,8 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
tok->inp = tok->buf + oldsize;
tok->end = tok->buf + newsize;
tok->start = start < 0 ? NULL : tok->buf + start;
+ tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
+ tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
}
return 1;
}
@@ -1883,6 +1887,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
while (end_quote_size != quote_size) {
c = tok_nextc(tok);
if (c == EOF || (quote_size == 1 && c == '\n')) {
+ assert(tok->multi_line_start != NULL);
// shift the tok_state's location into
// the start of string, and report the error
// from the initial quote character
More information about the Python-checkins
mailing list