[Python-checkins] bpo-43914: Correctly highlight SyntaxError exceptions for invalid generator expression in function calls (GH-28576)

[miss-islington]

https://github.com/python/cpython/commit/9e209d48cac35108f3955d3f610b6ce60b574ecc
commit: 9e209d48cac35108f3955d3f610b6ce60b574ecc
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-09-27T07:05:20-07:00
summary:

bpo-43914: Correctly highlight SyntaxError exceptions for invalid generator expression in function calls (GH-28576)

(cherry picked from commit e5f13ce5b48b551c09fdd0faeafa6ecf860de51c)

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

files:
M Grammar/python.gram
M Lib/test/test_syntax.py
M Parser/parser.c
M Parser/pegen.c
M Parser/pegen.h

diff --git a/Grammar/python.gram b/Grammar/python.gram
index b3ae906fd5414..76c0b4e67ec05 100644
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -833,7 +833,7 @@ invalid_arguments:
         RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") }
     | a=NAME b='=' expression for_if_clauses {
         RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")}
-    | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) }
+    | a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) }
     | args ',' a=expression b=for_if_clauses {
         RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") }
     | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) }
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index be8be898d0196..f9deb7b3313a7 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -1273,7 +1273,8 @@
 class SyntaxTestCase(unittest.TestCase):
 
     def _check_error(self, code, errtext,
-                     filename="<testcase>", mode="exec", subclass=None, lineno=None, offset=None):
+                     filename="<testcase>", mode="exec", subclass=None,
+                     lineno=None, offset=None, end_lineno=None, end_offset=None):
         """Check that compiling code raises SyntaxError with errtext.
 
         errtest is a regular expression that must be present in the
@@ -1293,6 +1294,11 @@ def _check_error(self, code, errtext,
                 self.assertEqual(err.lineno, lineno)
             if offset is not None:
                 self.assertEqual(err.offset, offset)
+            if end_lineno is not None:
+                self.assertEqual(err.end_lineno, end_lineno)
+            if end_offset is not None:
+                self.assertEqual(err.end_offset, end_offset)
+ 
         else:
             self.fail("compile() did not raise SyntaxError")
 
@@ -1432,6 +1438,11 @@ def test_kwargs_last3(self):
         self._check_error("int(**{'base': 10}, *['2'])",
                           "iterable argument unpacking follows "
                           "keyword argument unpacking")
+    
+    def test_generator_in_function_call(self):
+        self._check_error("foo(x,    y for y in range(3) for z in range(2) if z    , p)",
+                          "Generator expression must be parenthesized",
+                          lineno=1, end_lineno=1, offset=11, end_offset=53)
 
     def test_empty_line_after_linecont(self):
         # See issue-40847
diff --git a/Parser/parser.c b/Parser/parser.c
index e57c603e5ab76..2a437d5281b43 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -17902,15 +17902,15 @@ invalid_arguments_rule(Parser *p)
         }
         D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args for_if_clauses"));
         expr_ty a;
-        asdl_comprehension_seq* for_if_clauses_var;
+        asdl_comprehension_seq* b;
         if (
             (a = args_rule(p))  // args
             &&
-            (for_if_clauses_var = for_if_clauses_rule(p))  // for_if_clauses
+            (b = for_if_clauses_rule(p))  // for_if_clauses
         )
         {
             D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args for_if_clauses"));
-            _res = _PyPegen_nonparen_genexp_in_call ( p , a );
+            _res = _PyPegen_nonparen_genexp_in_call ( p , a , b );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 D(p->level--);
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 31ebccb9a8723..e20e926136828 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -2530,8 +2530,17 @@ void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
     return RAISE_SYNTAX_ERROR(msg);
 }
 
+
+static inline expr_ty
+_PyPegen_get_last_comprehension_item(comprehension_ty comprehension) {
+    if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) {
+        return comprehension->iter;
+    }
+    return PyPegen_last_item(comprehension->ifs, expr_ty);
+}
+
 void *
-_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args)
+_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions)
 {
     /* The rule that calls this function is 'args for_if_clauses'.
        For the input f(L, x for x in y), L and x are in args and
@@ -2545,8 +2554,11 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args)
         return NULL;
     }
 
-    return RAISE_SYNTAX_ERROR_STARTING_FROM(
+    comprehension_ty last_comprehension = PyPegen_last_item(comprehensions, comprehension_ty);
+
+    return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
         (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
+        _PyPegen_get_last_comprehension_item(last_comprehension),
         "Generator expression must be parenthesized"
     );
 }
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 4db79237532d8..40222d64499a9 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -325,7 +325,7 @@ _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
 }
 
 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
-void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args);
+void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
 
 
 // Generated function in parse.c - function definition in python.gram