[Python-checkins] [3.12] gh-105820: Fix tok_mode expression buffer in file & readline tokenizer (GH-105828) (#105832)

[lysnikolaou]

https://github.com/python/cpython/commit/e9cf5a324e10b693d464692ab19422c40d5e179f
commit: e9cf5a324e10b693d464692ab19422c40d5e179f
branch: 3.12
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: lysnikolaou <lisandrosnik at gmail.com>
date: 2023-06-15T17:21:58Z
summary:

[3.12] gh-105820: Fix tok_mode expression buffer in file & readline tokenizer (GH-105828) (#105832)

(cherry picked from commit d382ad49157b3802fc5619f68d96810def517869)

Co-authored-by: Lysandros Nikolaou <lisandrosnik at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst
M Lib/test/test_fstring.py
M Lib/test/test_tokenize.py
M Parser/tokenizer.c

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index cbb03080f797..8f6b576b5f78 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -15,7 +15,7 @@
 import unittest
 from test import support
 from test.support.os_helper import temp_cwd
-from test.support.script_helper import assert_python_failure
+from test.support.script_helper import assert_python_failure, assert_python_ok
 
 a_global = 'global variable'
 
@@ -1635,5 +1635,18 @@ def test_syntax_error_after_debug(self):
                                 "f'{1=}{1;}'",
                             ])
 
+    def test_debug_in_file(self):
+        with temp_cwd():
+            script = 'script.py'
+            with open('script.py', 'w') as f:
+                f.write(f"""\
+print(f'''{{
+3
+=}}''')""")
+
+            _, stdout, _ = assert_python_ok(script)
+        self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
+                         "3\n=3")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 15f53632cff8..5ad278499c9d 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -558,6 +558,19 @@ def test_string(self):
     OP         '}'           (1, 39) (1, 40)
     FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
     FSTRING_END "'"           (1, 52) (1, 53)
+    """)
+        self.check_tokenize("""\
+f'''{
+3
+=}'''""", """\
+    FSTRING_START "f'''"        (1, 0) (1, 4)
+    OP         '{'           (1, 4) (1, 5)
+    NL         '\\n'          (1, 5) (1, 6)
+    NUMBER     '3'           (2, 0) (2, 1)
+    NL         '\\n'          (2, 1) (2, 2)
+    OP         '='           (3, 0) (3, 1)
+    OP         '}'           (3, 1) (3, 2)
+    FSTRING_END "'''"         (3, 2) (3, 5)
     """)
 
     def test_function(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst
new file mode 100644
index 000000000000..407940add567
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst	
@@ -0,0 +1,3 @@
+Fix an f-string bug, where using a debug expression (the ``=`` sign) that
+appears in the last line of a file results to the debug buffer that holds the
+expression text being one character too small.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f41dd130af1f..1a59f542409f 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1039,9 +1039,6 @@ tok_readline_raw(struct tok_state *tok)
         if (line == NULL) {
             return 1;
         }
-        if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
-            return 0;
-        }
         if (tok->fp_interactive &&
             tok_concatenate_interactive_new_line(tok, line) == -1) {
             return 0;
@@ -1270,6 +1267,10 @@ tok_underflow_file(struct tok_state *tok) {
         tok->implicit_newline = 1;
     }
 
+    if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+
     ADVANCE_LINENO();
     if (tok->decoding_state != STATE_NORMAL) {
         if (tok->lineno > 2) {
@@ -1314,6 +1315,10 @@ tok_underflow_readline(struct tok_state* tok) {
         tok->implicit_newline = 1;
     }
 
+    if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+
     ADVANCE_LINENO();
     /* The default encoding is UTF-8, so make sure we don't have any
        non-UTF-8 sequences in it. */