Separate terminator token for f-string elements kind (#11842)

## Summary

This PR separates the terminator token for f-string elements depending
on the context. A list of f-string element can occur either in a regular
f-string or a format spec of an f-string. The terminator token is
different depending on that context.

## Test Plan

`cargo insta test` and verify the updated snapshots.

crates/ruff_python_parser/src/parser/expression.rs

@@ -18,7 +18,7 @@ use crate::string::{parse_fstring_literal_element, parse_string_literal, StringT
 use crate::token_set::TokenSet;
 use crate::{FStringErrorType, Mode, ParseErrorType, TokenKind};
 
-use super::{Parenthesized, RecoveryContextKind};
+use super::{FStringElementsKind, Parenthesized, RecoveryContextKind};
 
 /// A token set consisting of a newline or end of file.
 const NEWLINE_EOF_SET: TokenSet = TokenSet::new([TokenKind::Newline, TokenKind::EndOfFile]);
@@ -1307,7 +1307,7 @@ impl<'src> Parser<'src> {
         let flags = self.tokens.current_flags().as_any_string_flags();
 
         self.bump(TokenKind::FStringStart);
-        let elements = self.parse_fstring_elements(flags);
+        let elements = self.parse_fstring_elements(flags, FStringElementsKind::Regular);
 
         self.expect(TokenKind::FStringEnd);
 
@@ -1323,10 +1323,14 @@ impl<'src> Parser<'src> {
     /// # Panics
     ///
     /// If the parser isn't positioned at a `{` or `FStringMiddle` token.
-    fn parse_fstring_elements(&mut self, flags: ast::AnyStringFlags) -> FStringElements {
+    fn parse_fstring_elements(
+        &mut self,
+        flags: ast::AnyStringFlags,
+        kind: FStringElementsKind,
+    ) -> FStringElements {
         let mut elements = vec![];
 
-        self.parse_list(RecoveryContextKind::FStringElements, |parser| {
+        self.parse_list(RecoveryContextKind::FStringElements(kind), |parser| {
             let element = match parser.current_token_kind() {
                 TokenKind::Lbrace => {
                     FStringElement::Expression(parser.parse_fstring_expression_element(flags))
@@ -1463,7 +1467,7 @@ impl<'src> Parser<'src> {
 
         let format_spec = if self.eat(TokenKind::Colon) {
             let spec_start = self.node_start();
-            let elements = self.parse_fstring_elements(flags);
+            let elements = self.parse_fstring_elements(flags, FStringElementsKind::FormatSpec);
             Some(Box::new(ast::FStringFormatSpec {
                 range: self.node_range(spec_start),
                 elements,

crates/ruff_python_parser/src/parser/mod.rs

@@ -722,6 +722,37 @@ impl WithItemKind {
     }
 }
 
+#[derive(Debug, PartialEq, Copy, Clone)]
+enum FStringElementsKind {
+    /// The regular f-string elements.
+    ///
+    /// For example, the `"hello "`, `x`, and `" world"` elements in:
+    /// ```py
+    /// f"hello {x:.2f} world"
+    /// ```
+    Regular,
+
+    /// The f-string elements are part of the format specifier.
+    ///
+    /// For example, the `.2f` in:
+    /// ```py
+    /// f"hello {x:.2f} world"
+    /// ```
+    FormatSpec,
+}
+
+impl FStringElementsKind {
+    const fn list_terminator(self) -> TokenKind {
+        match self {
+            FStringElementsKind::Regular => TokenKind::FStringEnd,
+            // test_ok fstring_format_spec_terminator
+            // f"hello {x:} world"
+            // f"hello {x:.3f} world"
+            FStringElementsKind::FormatSpec => TokenKind::Rbrace,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Copy, Clone)]
 enum Parenthesized {
     /// The elements are parenthesized, e.g., `(a, b)`.
@@ -819,7 +850,7 @@ enum RecoveryContextKind {
 
     /// When parsing a list of f-string elements which are either literal elements
     /// or expressions.
-    FStringElements,
+    FStringElements(FStringElementsKind),
 }
 
 impl RecoveryContextKind {
@@ -954,10 +985,8 @@ impl RecoveryContextKind {
                     p.at(TokenKind::Colon)
                 }
             },
-            RecoveryContextKind::FStringElements => {
+            RecoveryContextKind::FStringElements(kind) => {
-                // Tokens other than `FStringEnd` and `}` are for better error recovery
+                p.at(kind.list_terminator())
-                p.at_ts(TokenSet::new([
-                    TokenKind::FStringEnd,
                     // test_err unterminated_fstring_newline_recovery
                     // f"hello
                     // 1 + 1
@@ -967,15 +996,7 @@ impl RecoveryContextKind {
                     // 3 + 3
                     // f"hello {x}
                     // 4 + 4
-                    TokenKind::Newline,
+                    || p.at(TokenKind::Newline)
-                    // When the parser is parsing f-string elements inside format spec,
-                    // the terminator would be `}`.
-
-                    // test_ok fstring_format_spec_terminator
-                    // f"hello {x:} world"
-                    // f"hello {x:.3f} world"
-                    TokenKind::Rbrace,
-                ]))
             }
         }
     }
@@ -1017,7 +1038,7 @@ impl RecoveryContextKind {
                 ) || p.at_name_or_soft_keyword()
             }
             RecoveryContextKind::WithItems(_) => p.at_expr(),
-            RecoveryContextKind::FStringElements => matches!(
+            RecoveryContextKind::FStringElements(_) => matches!(
                 p.current_token_kind(),
                 // Literal element
                 TokenKind::FStringMiddle
@@ -1111,9 +1132,14 @@ impl RecoveryContextKind {
                     "Expected an expression or the end of the with item list".to_string(),
                 ),
             },
-            RecoveryContextKind::FStringElements => ParseErrorType::OtherError(
+            RecoveryContextKind::FStringElements(kind) => match kind {
+                FStringElementsKind::Regular => ParseErrorType::OtherError(
                     "Expected an f-string element or the end of the f-string".to_string(),
                 ),
+                FStringElementsKind::FormatSpec => {
+                    ParseErrorType::OtherError("Expected an f-string element or a '}'".to_string())
+                }
+            },
         }
     }
 }
@@ -1152,6 +1178,7 @@ bitflags! {
         const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
         const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
         const F_STRING_ELEMENTS = 1 << 29;
+        const F_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 30;
     }
 }
 
@@ -1204,7 +1231,12 @@ impl RecoveryContext {
                 }
                 WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
             },
-            RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
+            RecoveryContextKind::FStringElements(kind) => match kind {
+                FStringElementsKind::Regular => RecoveryContext::F_STRING_ELEMENTS,
+                FStringElementsKind::FormatSpec => {
+                    RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC
+                }
+            },
         }
     }
 
@@ -1271,7 +1303,12 @@ impl RecoveryContext {
             RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
                 RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
             }
-            RecoveryContext::F_STRING_ELEMENTS => RecoveryContextKind::FStringElements,
+            RecoveryContext::F_STRING_ELEMENTS => {
+                RecoveryContextKind::FStringElements(FStringElementsKind::Regular)
+            }
+            RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC => {
+                RecoveryContextKind::FStringElements(FStringElementsKind::FormatSpec)
+            }
             _ => return None,
         })
     }

crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap

@@ -11,15 +11,15 @@ Module(
         body: [
             Expr(
                 StmtExpr {
-                    range: 0..14,
+                    range: 0..16,
                     value: FString(
                         ExprFString {
-                            range: 0..14,
+                            range: 0..16,
                             value: FStringValue {
                                 inner: Single(
                                     FString(
                                         FString {
-                                            range: 0..14,
+                                            range: 0..16,
                                             elements: [
                                                 Expression(
                                                     FStringExpressionElement {
@@ -110,17 +110,5 @@ Module(
 
   |
 1 | f"{lambda x: x}"
-  |               ^ Syntax Error: Expected FStringEnd, found '}'
+  |               ^ Syntax Error: Expected an f-string element or the end of the f-string
-  |
-
-
-  |
-1 | f"{lambda x: x}"
-  |                ^ Syntax Error: Expected a statement
-  |
-
-
-  |
-1 | f"{lambda x: x}"
-  |                 ^ Syntax Error: Expected a statement
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap

@@ -116,7 +116,7 @@ Module(
 
   |
 1 | f"hello {x:"
-  |            ^ Syntax Error: f-string: expecting '}'
+  |            ^ Syntax Error: Expected an f-string element or a '}'
 2 | f"hello {x:.3f"
   |
 
@@ -124,7 +124,7 @@ Module(
   |
 1 | f"hello {x:"
 2 | f"hello {x:.3f"
-  |               ^ Syntax Error: f-string: expecting '}'
+  |               ^ Syntax Error: Expected an f-string element or a '}'
   |
 
 

crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap

@@ -335,7 +335,7 @@ Module(
 4 | 2 + 2
 5 | f"hello {x:
 6 | 3 + 3
-  | ^ Syntax Error: Expected an f-string element or the end of the f-string
+  | ^ Syntax Error: Expected an f-string element or a '}'
 7 | f"hello {x}
 8 | 4 + 4
   |