language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-24 05:26:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Separate terminator token for f-string elements kind (#11842)

Browse source 
## Summary

This PR separates the terminator token for f-string elements depending
on the context. A list of f-string element can occur either in a regular
f-string or a format spec of an f-string. The terminator token is
different depending on that context.

## Test Plan

`cargo insta test` and verify the updated snapshots.
This commit is contained in:
Dhruv Manilawala 2024-06-12 13:57:35 +05:30 • committed by GitHub
parent 93973b96cb
commit a525b4be3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 73 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/ruff_python_parser/src/parser/expression.rs
View file

@ -18,7 +18,7 @@ use crate::string::{parse_fstring_literal_element, parse_string_literal, StringT
use crate::token_set::TokenSet;
use crate::{FStringErrorType, Mode, ParseErrorType, TokenKind};
use super::{Parenthesized, RecoveryContextKind};
use super::{FStringElementsKind, Parenthesized, RecoveryContextKind};
/// A token set consisting of a newline or end of file.
const NEWLINE_EOF_SET: TokenSet = TokenSet::new([TokenKind::Newline, TokenKind::EndOfFile]);
@ -1307,7 +1307,7 @@ impl<'src> Parser<'src> {
let flags = self.tokens.current_flags().as_any_string_flags();
self.bump(TokenKind::FStringStart);
let elements = self.parse_fstring_elements(flags);
let elements = self.parse_fstring_elements(flags, FStringElementsKind::Regular);
self.expect(TokenKind::FStringEnd);
@ -1323,10 +1323,14 @@ impl<'src> Parser<'src> {
/// # Panics
///
/// If the parser isn't positioned at a `{` or `FStringMiddle` token.
fn parse_fstring_elements(&mut self, flags: ast::AnyStringFlags) -> FStringElements {
fn parse_fstring_elements(
&mut self,
flags: ast::AnyStringFlags,
kind: FStringElementsKind,
) -> FStringElements {
let mut elements = vec![];
self.parse_list(RecoveryContextKind::FStringElements, |parser| {
self.parse_list(RecoveryContextKind::FStringElements(kind), |parser| {
let element = match parser.current_token_kind() {
TokenKind::Lbrace => {
FStringElement::Expression(parser.parse_fstring_expression_element(flags))
@ -1463,7 +1467,7 @@ impl<'src> Parser<'src> {
let format_spec = if self.eat(TokenKind::Colon) {
let spec_start = self.node_start();
let elements = self.parse_fstring_elements(flags);
let elements = self.parse_fstring_elements(flags, FStringElementsKind::FormatSpec);
Some(Box::new(ast::FStringFormatSpec {
range: self.node_range(spec_start),
elements,

77
crates/ruff_python_parser/src/parser/mod.rs
View file

@ -722,6 +722,37 @@ impl WithItemKind {
}
}
#[derive(Debug, PartialEq, Copy, Clone)]
enum FStringElementsKind {
/// The regular f-string elements.
///
/// For example, the `"hello "`, `x`, and `" world"` elements in:
/// ```py
/// f"hello {x:.2f} world"
/// ```
Regular,
/// The f-string elements are part of the format specifier.
///
/// For example, the `.2f` in:
/// ```py
/// f"hello {x:.2f} world"
/// ```
FormatSpec,
}
impl FStringElementsKind {
const fn list_terminator(self) -> TokenKind {
match self {
FStringElementsKind::Regular => TokenKind::FStringEnd,
// test_ok fstring_format_spec_terminator
// f"hello {x:} world"
// f"hello {x:.3f} world"
FStringElementsKind::FormatSpec => TokenKind::Rbrace,
}
}
}
#[derive(Debug, PartialEq, Copy, Clone)]
enum Parenthesized {
/// The elements are parenthesized, e.g., `(a, b)`.
@ -819,7 +850,7 @@ enum RecoveryContextKind {
/// When parsing a list of f-string elements which are either literal elements
/// or expressions.
FStringElements,
FStringElements(FStringElementsKind),
}
impl RecoveryContextKind {
@ -954,10 +985,8 @@ impl RecoveryContextKind {
p.at(TokenKind::Colon)
}
},
RecoveryContextKind::FStringElements => {
// Tokens other than `FStringEnd` and `}` are for better error recovery
p.at_ts(TokenSet::new([
TokenKind::FStringEnd,
RecoveryContextKind::FStringElements(kind) => {
p.at(kind.list_terminator())
// test_err unterminated_fstring_newline_recovery
// f"hello
// 1 + 1
@ -967,15 +996,7 @@ impl RecoveryContextKind {
// 3 + 3
// f"hello {x}
// 4 + 4
TokenKind::Newline,
// When the parser is parsing f-string elements inside format spec,
// the terminator would be `}`.
// test_ok fstring_format_spec_terminator
// f"hello {x:} world"
// f"hello {x:.3f} world"
TokenKind::Rbrace,
]))
|| p.at(TokenKind::Newline)
}
}
}
@ -1017,7 +1038,7 @@ impl RecoveryContextKind {
) || p.at_name_or_soft_keyword()
}
RecoveryContextKind::WithItems(_) => p.at_expr(),
RecoveryContextKind::FStringElements => matches!(
RecoveryContextKind::FStringElements(_) => matches!(
p.current_token_kind(),
// Literal element
TokenKind::FStringMiddle
@ -1111,9 +1132,14 @@ impl RecoveryContextKind {
"Expected an expression or the end of the with item list".to_string(),
),
},
RecoveryContextKind::FStringElements => ParseErrorType::OtherError(
"Expected an f-string element or the end of the f-string".to_string(),
),
RecoveryContextKind::FStringElements(kind) => match kind {
FStringElementsKind::Regular => ParseErrorType::OtherError(
"Expected an f-string element or the end of the f-string".to_string(),
),
FStringElementsKind::FormatSpec => {
ParseErrorType::OtherError("Expected an f-string element or a '}'".to_string())
}
},
}
}
}
@ -1152,6 +1178,7 @@ bitflags! {
const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
const F_STRING_ELEMENTS = 1 << 29;
const F_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 30;
}
}
@ -1204,7 +1231,12 @@ impl RecoveryContext {
}
WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
},
RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
RecoveryContextKind::FStringElements(kind) => match kind {
FStringElementsKind::Regular => RecoveryContext::F_STRING_ELEMENTS,
FStringElementsKind::FormatSpec => {
RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC
}
},
}
}
@ -1271,7 +1303,12 @@ impl RecoveryContext {
RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
}
RecoveryContext::F_STRING_ELEMENTS => RecoveryContextKind::FStringElements,
RecoveryContext::F_STRING_ELEMENTS => {
RecoveryContextKind::FStringElements(FStringElementsKind::Regular)
}
RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC => {
RecoveryContextKind::FStringElements(FStringElementsKind::FormatSpec)
}
_ => return None,
})
}

20
crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap
View file

@ -11,15 +11,15 @@ Module(
body: [
Expr(
StmtExpr {
range: 0..14,
range: 0..16,
value: FString(
ExprFString {
range: 0..14,
range: 0..16,
value: FStringValue {
inner: Single(
FString(
FString {
range: 0..14,
range: 0..16,
elements: [
Expression(
FStringExpressionElement {
@ -110,17 +110,5 @@ Module(
|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected FStringEnd, found '}'
|
|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected a statement
|
|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected a statement
| ^ Syntax Error: Expected an f-string element or the end of the f-string
|

4
crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap
View file

@ -116,7 +116,7 @@ Module(
|
1 | f"hello {x:"
| ^ Syntax Error: f-string: expecting '}'
| ^ Syntax Error: Expected an f-string element or a '}'
2 | f"hello {x:.3f"
|
@ -124,7 +124,7 @@ Module(
|
1 | f"hello {x:"
2 | f"hello {x:.3f"
| ^ Syntax Error: f-string: expecting '}'
| ^ Syntax Error: Expected an f-string element or a '}'
|

2
crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap
View file

@ -335,7 +335,7 @@ Module(
4 | 2 + 2
5 | f"hello {x:
6 | 3 + 3
| ^ Syntax Error: Expected an f-string element or the end of the f-string
| ^ Syntax Error: Expected an f-string element or a '}'
7 | f"hello {x}
8 | 4 + 4
|