[Python-checkins] gh-105194: Fix format specifier escaped characters in f-strings (#105231)

[pablogsal]

https://github.com/python/cpython/commit/41de54378d54f7ffc38f07db4219e80f48c4249e
commit: 41de54378d54f7ffc38f07db4219e80f48c4249e
branch: main
author: Pablo Galindo Salgado <Pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2023-06-02T13:33:26+02:00
summary:

gh-105194: Fix format specifier escaped characters in f-strings (#105231)

files:
A Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst
M Grammar/python.gram
M Lib/test/test_fstring.py
M Parser/action_helpers.c
M Parser/parser.c
M Parser/pegen.h

diff --git a/Grammar/python.gram b/Grammar/python.gram
index 9131835f7421b..6b2a46aff0dcf 100644
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -923,7 +923,7 @@ fstring_conversion[ResultTokenWithMetadata*]:
 fstring_full_format_spec[ResultTokenWithMetadata*]:
     | colon=':' spec=fstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
 fstring_format_spec[expr_ty]:
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+    | t=FSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
     | fstring_replacement_field
 fstring[expr_ty]:
     | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 3ba2f943e6e96..031b94d8d58a3 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -764,6 +764,16 @@ def test_format_specifier_expressions(self):
                              """f'{"s"!{"r"}}'""",
                              ])
 
+    def test_custom_format_specifier(self):
+        class CustomFormat:
+            def __format__(self, format_spec):
+                return format_spec
+
+        self.assertEqual(f'{CustomFormat():\n}', '\n')
+        self.assertEqual(f'{CustomFormat():\u2603}', '☃')
+        with self.assertWarns(SyntaxWarning):
+            exec('f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat})
+
     def test_side_effect_order(self):
         class X:
             def __init__(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst
new file mode 100644
index 0000000000000..adee74f5894b5
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst	
@@ -0,0 +1,2 @@
+Do not escape with backslashes f-string format specifiers. Patch by Pablo
+Galindo
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 2411da2c705e7..7786124fde82d 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1350,6 +1350,25 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
                             p->arena);
 }
 
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
+    Py_ssize_t bsize;
+    char* bstr;
+    if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
+        return NULL;
+    }
+    PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok);
+    if (str == NULL) {
+        return NULL;
+    }
+    if (_PyArena_AddPyObject(p->arena, str) < 0) {
+        Py_DECREF(str);
+        return NULL;
+    }
+    return _PyAST_Constant(str, NULL, tok->lineno, tok->col_offset,
+                           tok->end_lineno, tok->end_col_offset,
+                           p->arena);
+}
+
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
     char* bstr = PyBytes_AsString(tok->bytes);
     if (bstr == NULL) {
diff --git a/Parser/parser.c b/Parser/parser.c
index 1705ebd456b5f..006ee297974a6 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -16323,7 +16323,7 @@ fstring_format_spec_rule(Parser *p)
         )
         {
             D(fprintf(stderr, "%*c+ fstring_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE"));
-            _res = _PyPegen_constant_from_token ( p , t );
+            _res = _PyPegen_decoded_constant_from_token ( p , t );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 p->level--;
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 8800e9f97f5e0..fe13d10e6b83e 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -328,6 +328,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
                      int lineno, int col_offset, int end_lineno,
                      int end_col_offset, PyArena *arena);
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
 expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);