[Python-checkins] Fix tokenizer error when raw decoding null bytes (GH-25080)

[pablogsal]

https://github.com/python/cpython/commit/92a02c1f7e2dcdc62913a4236589e7e5d96172b9
commit: 92a02c1f7e2dcdc62913a4236589e7e5d96172b9
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-03-30T00:24:49+01:00
summary:

Fix tokenizer error when raw decoding null bytes (GH-25080)

files:
M Parser/tokenizer.c

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index d18fffaf3dbda..ad32293d70b78 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -813,6 +813,9 @@ tok_readline_raw(struct tok_state *tok)
             tok_concatenate_interactive_new_line(tok, line) == -1) {
             return 0;
         }
+        if (*tok->inp == '\0') {
+            return 0;
+        }
         tok->inp = strchr(tok->inp, '\0');
     } while (tok->inp[-1] != '\n');
     return 1;
@@ -963,7 +966,7 @@ tok_underflow_file(struct tok_state *tok) {
         if (tok->lineno > 2) {
             tok->decoding_state = STATE_NORMAL;
         }
-        else if (!check_coding_spec(tok->cur, tok->end - tok->cur,
+        else if (!check_coding_spec(tok->cur, strlen(tok->cur),
                                     tok, fp_setreadl))
         {
             return 0;