[Python-checkins] bpo-46237: Fix the line number of tokenizer errors inside f-strings (GH-30463)

[miss-islington]

https://github.com/python/cpython/commit/19a85501cee24a6e426a431243d0adcb5664c6fe
commit: 19a85501cee24a6e426a431243d0adcb5664c6fe
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2022-01-11T08:33:08-08:00
summary:

bpo-46237: Fix the line number of tokenizer errors inside f-strings (GH-30463)

(cherry picked from commit 6fa8b2ceee38187b0ae96aee12fe4f0a5c8a2ce7)

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2022-01-07-19-33-05.bpo-46237.9A6Hpq.rst
M Lib/test/test_exceptions.py
M Parser/pegen.c
M Parser/string_parser.c

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index cc0640dda0980..86b5dccaaed98 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -266,6 +266,18 @@ def baz():
         check("(1+)", 1, 4)
         check("[interesting\nfoo()\n", 1, 1)
         check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1)
+        check("""f'''
+            {
+            (123_a)
+            }'''""", 3, 17)
+        check("""f'''
+            {
+            f\"\"\"
+            {
+            (123_a)
+            }
+            \"\"\"
+            }'''""", 5, 17)
 
         # Errors thrown by symtable.c
         check('x = [(yield i) for i in range(3)]', 1, 7)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-07-19-33-05.bpo-46237.9A6Hpq.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-07-19-33-05.bpo-46237.9A6Hpq.rst
new file mode 100644
index 0000000000000..931a2603293c3
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-07-19-33-05.bpo-46237.9A6Hpq.rst	
@@ -0,0 +1,2 @@
+Fix the line number of tokenizer errors inside f-strings. Patch by Pablo
+Galindo.
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 0504906c947d0..e507415f6d14c 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -701,10 +701,10 @@ initialize_token(Parser *p, Token *token, const char *start, const char *end, in
     int col_offset = (start != NULL && start >= line_start) ? (int)(start - line_start) : -1;
     int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
 
-    token->lineno = p->starting_lineno + lineno;
-    token->col_offset = p->tok->lineno == 1 ? p->starting_col_offset + col_offset : col_offset;
-    token->end_lineno = p->starting_lineno + end_lineno;
-    token->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset;
+    token->lineno = lineno;
+    token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + col_offset : col_offset;
+    token->end_lineno = end_lineno;
+    token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + end_col_offset : end_col_offset;
 
     p->fill += 1;
 
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index dcd298cb358ee..c83e63fc6f8f2 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -392,11 +392,14 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
         return NULL;
     }
     Py_INCREF(p->tok->filename);
+
     tok->filename = p->tok->filename;
+    tok->lineno = t->lineno + lines - 1;
 
     Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, p->flags, p->feature_version,
                                      NULL, p->arena);
-    p2->starting_lineno = t->lineno + lines - 1;
+
+    p2->starting_lineno = t->lineno + lines;
     p2->starting_col_offset = t->col_offset + cols;
 
     expr = _PyPegen_run_parser(p2);