[Python-checkins] bpo-42864: Simplify the tokenizer exceptions after generic SyntaxError (GH-24273)

[miss-islington]

https://github.com/python/cpython/commit/c3f167d7b243f8b8e1b797586e6cef35add013bc
commit: c3f167d7b243f8b8e1b797586e6cef35add013bc
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-01-20T11:11:56-08:00
summary:

bpo-42864: Simplify the tokenizer exceptions after generic SyntaxError (GH-24273)



Automerge-Triggered-By: GH:pablogsal

files:
M Parser/pegen.c

diff --git a/Parser/pegen.c b/Parser/pegen.c
index 6c27980602105..0d39030ea6ed1 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1177,14 +1177,9 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         return 0;
     }
 
-
     Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
     Py_ssize_t current_err_line = current_token->lineno;
 
-    // Save the tokenizer state to restore them later in case we found nothing
-    struct tok_state saved_tok;
-    memcpy(&saved_tok, p->tok, sizeof(struct tok_state));
-
     for (;;) {
         const char *start;
         const char *end;
@@ -1206,8 +1201,6 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         break;
     }
 
-    // Restore the tokenizer state
-    memcpy(p->tok, &saved_tok, sizeof(struct tok_state));
     return 0;
 }
 
@@ -1239,10 +1232,10 @@ _PyPegen_run_parser(Parser *p)
                 RAISE_INDENTATION_ERROR("unexpected unindent");
             }
             else {
-                if (_PyPegen_check_tokenizer_errors(p)) {
-                    return NULL;
-                }
                 RAISE_SYNTAX_ERROR("invalid syntax");
+                // _PyPegen_check_tokenizer_errors will override the existing
+                // generic SyntaxError we just raised if errors are found.
+                _PyPegen_check_tokenizer_errors(p);
             }
         }
         return NULL;