[Python-checkins] gh-104016: Fixed off by 1 error in f string tokenizer (#104047)

[pablogsal]

https://github.com/python/cpython/commit/5078eedc5b18f0d208af6e30f60b33419132d1b6
commit: 5078eedc5b18f0d208af6e30f60b33419132d1b6
branch: main
author: jx124 <64946984+jx124 at users.noreply.github.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2023-05-01T19:15:47Z
summary:

gh-104016: Fixed off by 1 error in f string tokenizer (#104047)

Co-authored-by: sunmy2019 <59365878+sunmy2019 at users.noreply.github.com>
Co-authored-by: Ken Jin <kenjin at python.org>
Co-authored-by: Pablo Galindo <pablogsal at gmail.com>

files:
M Lib/test/test_fstring.py
M Parser/tokenizer.c
M Parser/tokenizer.h

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 5e94c99ae65a..5c5176dc54a6 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -565,7 +565,23 @@ def test_fstring_nested_too_deeply(self):
         self.assertAllRaise(SyntaxError,
                             "f-string: expressions nested too deeply",
                             ['f"{1+2:{1+2:{1+1:{1}}}}"'])
+        
+        def create_nested_fstring(n):
+            if n == 0:
+                return "1+1"
+            prev = create_nested_fstring(n-1)
+            return f'f"{{{prev}}}"'
 
+        self.assertAllRaise(SyntaxError,
+                            "too many nested f-strings",
+                            [create_nested_fstring(160)])
+    
+    def test_syntax_error_in_nested_fstring(self):
+        # See gh-104016 for more information on this crash
+        self.assertAllRaise(SyntaxError,
+                            "invalid syntax",
+                            ['f"{1 1:' + ('{f"1:' * 199)])
+    
     def test_double_braces(self):
         self.assertEqual(f'{{', '{')
         self.assertEqual(f'a{{', 'a{')
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 8fb9be7bfd01..d2f9fee110eb 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -43,12 +43,12 @@
 #ifdef Py_DEBUG
 static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
     assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index < MAXLEVEL);
+    assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
     return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
 }
 static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
     assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index < MAXLEVEL);
+    assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); 
     return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
 }
 #else
@@ -2235,6 +2235,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
 
         p_start = tok->start;
         p_end = tok->cur;
+        if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
+            return MAKE_TOKEN(syntaxerror(tok, "too many nested f-strings"));
+        }
         tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
         the_current_tok->kind = TOK_FSTRING_MODE;
         the_current_tok->f_string_quote = quote;
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 8b4213c4ce3b..5e2171885ac7 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -10,8 +10,9 @@ extern "C" {
 
 #include "pycore_token.h" /* For token types */
 
-#define MAXINDENT 100   /* Max indentation level */
-#define MAXLEVEL 200    /* Max parentheses level */
+#define MAXINDENT 100       /* Max indentation level */
+#define MAXLEVEL 200        /* Max parentheses level */
+#define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */
 
 enum decoding_state {
     STATE_INIT,
@@ -123,7 +124,7 @@ struct tok_state {
     enum interactive_underflow_t interactive_underflow;
     int report_warnings;
     // TODO: Factor this into its own thing
-    tokenizer_mode tok_mode_stack[MAXLEVEL];
+    tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
     int tok_mode_stack_index;
     int tok_report_warnings;
 #ifdef Py_DEBUG