[Python-checkins] gh-96611: Fix error message for invalid UTF-8 in mid-multiline string (GH-96623)

[miss-islington]

https://github.com/python/cpython/commit/b6af9337163e03cc853aa15905658748abef0901
commit: b6af9337163e03cc853aa15905658748abef0901
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2022-09-06T16:36:03-07:00
summary:

gh-96611: Fix error message for invalid UTF-8 in mid-multiline string (GH-96623)

(cherry picked from commit 05692c67c51b78a5a5a7bb61d646519025e38015)

Co-authored-by: Michael Droettboom <mdboom at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2022-09-06-16-22-13.gh-issue-96611.14wIX8.rst
M Lib/test/test_source_encoding.py
M Parser/tokenizer.c

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index a0cb605c1651..219c25cd2f6c 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -147,6 +147,18 @@ def test_error_from_string(self):
         self.assertTrue(c.exception.args[0].startswith(expected),
                         msg=c.exception.args[0])
 
+    def test_file_parse_error_multiline(self):
+        # gh96611:
+        with open(TESTFN, "wb") as fd:
+            fd.write(b'print("""\n\xb1""")\n')
+
+        try:
+            retcode, stdout, stderr = script_helper.assert_python_failure(TESTFN)
+
+            self.assertGreater(retcode, 0)
+            self.assertIn(b"Non-UTF-8 code starting with '\\xb1'", stderr)
+        finally:
+            os.unlink(TESTFN)
 
 class AbstractSourceEncodingTest:
 
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-06-16-22-13.gh-issue-96611.14wIX8.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-06-16-22-13.gh-issue-96611.14wIX8.rst
new file mode 100644
index 000000000000..08bd409bc9f9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-06-16-22-13.gh-issue-96611.14wIX8.rst	
@@ -0,0 +1,2 @@
+When loading a file with invalid UTF-8 inside a multi-line string, a correct
+SyntaxError is emitted.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 579474c43354..2f00d208fea9 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1965,6 +1965,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         /* Get rest of string */
         while (end_quote_size != quote_size) {
             c = tok_nextc(tok);
+            if (tok->done == E_DECODE)
+                break;
             if (c == EOF || (quote_size == 1 && c == '\n')) {
                 assert(tok->multi_line_start != NULL);
                 // shift the tok_state's location into