Index multiline f-strings (#9837)

Fix #9777.
This commit is contained in:
Seo Sanghyeon 2024-02-06 11:25:33 +09:00 committed by GitHub
parent 83195a6030
commit df7fb95cbc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 180 additions and 25 deletions

View file

@ -675,6 +675,7 @@ impl<'source> Lexer<'source> {
Ok(Some(Tok::FStringMiddle {
value,
is_raw: fstring.is_raw_string(),
triple_quoted: fstring.is_triple_quoted(),
}))
}

View file

@ -1622,7 +1622,7 @@ FStringExpr: StringType = {
FStringMiddlePattern: ast::FStringElement = {
FStringReplacementField,
<location:@L> <fstring_middle:fstring_middle> <end_location:@R> =>? {
let (source, is_raw) = fstring_middle;
let (source, is_raw, _) = fstring_middle;
Ok(parse_fstring_literal_element(&source, is_raw, (location..end_location).into())?)
}
};
@ -2067,7 +2067,8 @@ extern {
},
fstring_middle => token::Tok::FStringMiddle {
value: <String>,
is_raw: <bool>
is_raw: <bool>,
triple_quoted: <bool>
},
name => token::Tok::Name { name: <String> },
ipy_escape_command => token::Tok::IpyEscapeCommand {

View file

@ -1,5 +1,5 @@
// auto-generated: "lalrpop 0.20.0"
// sha3: 031689e389556292d9dbd8a1b1ff8ca29bac76d83f1b345630481d620b89e1c2
// sha3: aa0540221d25f4eadfc9e043fb4fc631d537b672b8a96785dfec2407e0524b79
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
use crate::{
@ -50,7 +50,7 @@ mod __parse__Top {
Variant0(token::Tok),
Variant1((f64, f64)),
Variant2(f64),
Variant3((String, bool)),
Variant3((String, bool, bool)),
Variant4(Int),
Variant5((IpyEscapeKind, String)),
Variant6(String),
@ -151,7 +151,7 @@ mod __parse__Top {
Variant101(ast::TypeParams),
Variant102(core::option::Option<ast::TypeParams>),
Variant103(ast::UnaryOp),
Variant104(core::option::Option<(String, bool)>),
Variant104(core::option::Option<(String, bool, bool)>),
}
const __ACTION: &[i16] = &[
// State 0
@ -6090,7 +6090,7 @@ mod __parse__Top {
token::Tok::StartModule if true => Some(93),
token::Tok::Complex { real: _, imag: _ } if true => Some(94),
token::Tok::Float { value: _ } if true => Some(95),
token::Tok::FStringMiddle { value: _, is_raw: _ } if true => Some(96),
token::Tok::FStringMiddle { value: _, is_raw: _, triple_quoted: _ } if true => Some(96),
token::Tok::Int { value: _ } if true => Some(97),
token::Tok::IpyEscapeCommand { kind: _, value: _ } if true => Some(98),
token::Tok::Name { name: _ } if true => Some(99),
@ -6116,7 +6116,7 @@ mod __parse__Top {
_ => unreachable!(),
},
96 => match __token {
token::Tok::FStringMiddle { value: __tok0, is_raw: __tok1 } if true => __Symbol::Variant3((__tok0, __tok1)),
token::Tok::FStringMiddle { value: __tok0, is_raw: __tok1, triple_quoted: __tok2 } if true => __Symbol::Variant3((__tok0, __tok1, __tok2)),
_ => unreachable!(),
},
97 => match __token {
@ -18386,7 +18386,7 @@ mod __parse__Top {
fn __pop_Variant3<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (String, bool), TextSize)
) -> (TextSize, (String, bool, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
@ -19136,7 +19136,7 @@ mod __parse__Top {
fn __pop_Variant104<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, core::option::Option<(String, bool)>, TextSize)
) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
@ -35910,7 +35910,7 @@ fn __action184<
(_, parameters, _): (TextSize, core::option::Option<ast::Parameters>, TextSize),
(_, end_location_args, _): (TextSize, TextSize, TextSize),
(_, _, _): (TextSize, token::Tok, TextSize),
(_, fstring_middle, _): (TextSize, core::option::Option<(String, bool)>, TextSize),
(_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
(_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@ -36413,12 +36413,12 @@ fn __action220<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, fstring_middle, _): (TextSize, (String, bool), TextSize),
(_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
{
let (source, is_raw) = fstring_middle;
let (source, is_raw, _) = fstring_middle;
Ok(parse_fstring_literal_element(&source, is_raw, (location..end_location).into())?)
}
}
@ -37357,8 +37357,8 @@ fn __action281<
>(
source_code: &str,
mode: Mode,
(_, __0, _): (TextSize, (String, bool), TextSize),
) -> core::option::Option<(String, bool)>
(_, __0, _): (TextSize, (String, bool, bool), TextSize),
) -> core::option::Option<(String, bool, bool)>
{
Some(__0)
}
@ -37371,7 +37371,7 @@ fn __action282<
mode: Mode,
__lookbehind: &TextSize,
__lookahead: &TextSize,
) -> core::option::Option<(String, bool)>
) -> core::option::Option<(String, bool, bool)>
{
None
}
@ -48505,7 +48505,7 @@ fn __action804<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, bool), TextSize),
__0: (TextSize, (String, bool, bool), TextSize),
__1: (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -49609,7 +49609,7 @@ fn __action839<
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, TextSize, TextSize),
__3: (TextSize, token::Tok, TextSize),
__4: (TextSize, core::option::Option<(String, bool)>, TextSize),
__4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__5: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
__6: (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@ -64527,7 +64527,7 @@ fn __action1315<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, bool), TextSize),
__0: (TextSize, (String, bool, bool), TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.2;
@ -65430,7 +65430,7 @@ fn __action1347<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool)>, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -77662,7 +77662,7 @@ fn __action1727<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool)>, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -77693,7 +77693,7 @@ fn __action1728<
mode: Mode,
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize),
__2: (TextSize, core::option::Option<(String, bool)>, TextSize),
__2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -79598,7 +79598,7 @@ fn __action1785<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, (String, bool), TextSize),
__3: (TextSize, (String, bool, bool), TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -79661,7 +79661,7 @@ fn __action1787<
mode: Mode,
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize),
__2: (TextSize, (String, bool), TextSize),
__2: (TextSize, (String, bool, bool), TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "normal ",
is_raw: false,
triple_quoted: false,
},
2..9,
),
@ -32,6 +33,7 @@ expression: lex_source(source)
FStringMiddle {
value: " {another} ",
is_raw: false,
triple_quoted: false,
},
14..27,
),
@ -53,6 +55,7 @@ expression: lex_source(source)
FStringMiddle {
value: " {",
is_raw: false,
triple_quoted: false,
},
32..35,
),
@ -74,6 +77,7 @@ expression: lex_source(source)
FStringMiddle {
value: "}",
is_raw: false,
triple_quoted: false,
},
42..44,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\n# not a comment ",
is_raw: false,
triple_quoted: true,
},
4..21,
),
@ -46,6 +47,7 @@ expression: lex_source(source)
FStringMiddle {
value: " # not a comment\n",
is_raw: false,
triple_quoted: true,
},
42..59,
),

View file

@ -35,6 +35,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
7..8,
),
@ -70,6 +71,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
14..15,
),
@ -91,6 +93,7 @@ expression: lex_source(source)
FStringMiddle {
value: ".3f!r",
is_raw: false,
triple_quoted: false,
},
18..23,
),
@ -102,6 +105,7 @@ expression: lex_source(source)
FStringMiddle {
value: " {x!r}",
is_raw: false,
triple_quoted: false,
},
24..32,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\",
is_raw: false,
triple_quoted: false,
},
2..3,
),
@ -32,6 +33,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\\"\\",
is_raw: false,
triple_quoted: false,
},
6..9,
),
@ -57,6 +59,7 @@ expression: lex_source(source)
FStringMiddle {
value: " \\\"\\\"\\\n end",
is_raw: false,
triple_quoted: false,
},
13..24,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\",
is_raw: false,
triple_quoted: false,
},
2..3,
),
@ -40,6 +41,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\\\",
is_raw: false,
triple_quoted: false,
},
12..14,
),
@ -69,6 +71,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\{foo}",
is_raw: false,
triple_quoted: false,
},
23..31,
),
@ -84,6 +87,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\\\{foo}",
is_raw: false,
triple_quoted: false,
},
35..44,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\",
is_raw: true,
triple_quoted: false,
},
3..4,
),
@ -32,6 +33,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\\"\\",
is_raw: true,
triple_quoted: false,
},
7..10,
),
@ -57,6 +59,7 @@ expression: lex_source(source)
FStringMiddle {
value: " \\\"\\\"\\\n end",
is_raw: true,
triple_quoted: false,
},
14..25,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "first ",
is_raw: false,
triple_quoted: false,
},
2..8,
),
@ -58,6 +59,7 @@ expression: lex_source(source)
FStringMiddle {
value: " second",
is_raw: false,
triple_quoted: false,
},
41..48,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\nhello\n world\n",
is_raw: false,
triple_quoted: true,
},
4..21,
),
@ -26,6 +27,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\n world\nhello\n",
is_raw: false,
triple_quoted: true,
},
29..46,
),
@ -41,6 +43,7 @@ expression: lex_source(source)
FStringMiddle {
value: "some ",
is_raw: false,
triple_quoted: false,
},
52..57,
),
@ -56,6 +59,7 @@ expression: lex_source(source)
FStringMiddle {
value: "multiline\nallowed ",
is_raw: false,
triple_quoted: true,
},
62..80,
),
@ -85,6 +89,7 @@ expression: lex_source(source)
FStringMiddle {
value: " string",
is_raw: false,
triple_quoted: false,
},
87..94,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\N{BULLET} normal \\Nope \\N",
is_raw: false,
triple_quoted: false,
},
2..28,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\N",
is_raw: true,
triple_quoted: false,
},
3..5,
),
@ -32,6 +33,7 @@ expression: lex_source(source)
FStringMiddle {
value: " normal",
is_raw: true,
triple_quoted: false,
},
13..20,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "foo ",
is_raw: false,
triple_quoted: false,
},
2..6,
),
@ -26,6 +27,7 @@ expression: lex_source(source)
FStringMiddle {
value: "bar ",
is_raw: false,
triple_quoted: false,
},
9..13,
),
@ -81,6 +83,7 @@ expression: lex_source(source)
FStringMiddle {
value: " baz",
is_raw: false,
triple_quoted: false,
},
29..33,
),
@ -96,6 +99,7 @@ expression: lex_source(source)
FStringMiddle {
value: "foo ",
is_raw: false,
triple_quoted: false,
},
37..41,
),
@ -111,6 +115,7 @@ expression: lex_source(source)
FStringMiddle {
value: "bar",
is_raw: false,
triple_quoted: false,
},
44..47,
),
@ -126,6 +131,7 @@ expression: lex_source(source)
FStringMiddle {
value: " some ",
is_raw: false,
triple_quoted: false,
},
49..55,
),
@ -141,6 +147,7 @@ expression: lex_source(source)
FStringMiddle {
value: "another",
is_raw: false,
triple_quoted: false,
},
58..65,
),

View file

@ -27,6 +27,7 @@ expression: lex_source(source)
FStringMiddle {
value: "{}",
is_raw: false,
triple_quoted: false,
},
8..12,
),
@ -42,6 +43,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
16..17,
),
@ -65,6 +67,7 @@ expression: lex_source(source)
FStringMiddle {
value: "{",
is_raw: false,
triple_quoted: false,
},
23..25,
),
@ -80,6 +83,7 @@ expression: lex_source(source)
FStringMiddle {
value: "}",
is_raw: false,
triple_quoted: false,
},
27..29,
),
@ -95,6 +99,7 @@ expression: lex_source(source)
FStringMiddle {
value: "{{}}",
is_raw: false,
triple_quoted: false,
},
33..41,
),
@ -110,6 +115,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
45..46,
),
@ -125,6 +131,7 @@ expression: lex_source(source)
FStringMiddle {
value: " {} {",
is_raw: false,
triple_quoted: false,
},
48..56,
),
@ -140,6 +147,7 @@ expression: lex_source(source)
FStringMiddle {
value: "} {{}} ",
is_raw: false,
triple_quoted: false,
},
58..71,
),

View file

@ -11,6 +11,7 @@ expression: fstring_single_quote_escape_eol(MAC_EOL)
FStringMiddle {
value: "text \\\r more text",
is_raw: false,
triple_quoted: false,
},
2..19,
),

View file

@ -11,6 +11,7 @@ expression: fstring_single_quote_escape_eol(UNIX_EOL)
FStringMiddle {
value: "text \\\n more text",
is_raw: false,
triple_quoted: false,
},
2..19,
),

View file

@ -11,6 +11,7 @@ expression: fstring_single_quote_escape_eol(WINDOWS_EOL)
FStringMiddle {
value: "text \\\r\n more text",
is_raw: false,
triple_quoted: false,
},
2..20,
),

View file

@ -29,6 +29,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
8..9,
),
@ -64,6 +65,7 @@ expression: lex_source(source)
FStringMiddle {
value: ".3f",
is_raw: false,
triple_quoted: false,
},
15..18,
),
@ -75,6 +77,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
19..20,
),
@ -96,6 +99,7 @@ expression: lex_source(source)
FStringMiddle {
value: ".",
is_raw: false,
triple_quoted: false,
},
23..24,
),
@ -117,6 +121,7 @@ expression: lex_source(source)
FStringMiddle {
value: "f",
is_raw: false,
triple_quoted: false,
},
27..28,
),
@ -128,6 +133,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
29..30,
),
@ -151,6 +157,7 @@ expression: lex_source(source)
FStringMiddle {
value: "*^",
is_raw: false,
triple_quoted: false,
},
34..36,
),
@ -194,6 +201,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
44..45,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "foo ",
is_raw: false,
triple_quoted: false,
},
2..6,
),
@ -36,6 +37,7 @@ expression: lex_source(source)
FStringMiddle {
value: " bar",
is_raw: false,
triple_quoted: false,
},
12..16,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: true,
},
4..6,
),
@ -36,6 +37,7 @@ expression: lex_source(source)
FStringMiddle {
value: "d\n",
is_raw: false,
triple_quoted: true,
},
14..16,
),
@ -47,6 +49,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: true,
},
17..19,
),
@ -66,6 +69,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: true,
},
27..29,
),
@ -91,6 +95,7 @@ expression: lex_source(source)
FStringMiddle {
value: "a\n b\n c\n",
is_raw: false,
triple_quoted: true,
},
37..61,
),
@ -102,6 +107,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: true,
},
62..64,
),
@ -121,6 +127,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: false,
},
70..72,
),
@ -146,6 +153,7 @@ expression: lex_source(source)
FStringMiddle {
value: "d",
is_raw: false,
triple_quoted: false,
},
80..81,
),
@ -161,6 +169,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: false,
},
83..85,
),
@ -180,6 +189,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: false,
},
89..91,
),
@ -205,6 +215,7 @@ expression: lex_source(source)
FStringMiddle {
value: "a",
is_raw: false,
triple_quoted: false,
},
99..100,
),
@ -230,6 +241,7 @@ expression: lex_source(source)
FStringMiddle {
value: "__",
is_raw: false,
triple_quoted: false,
},
112..114,
),

View file

@ -25,6 +25,7 @@ expression: lex_source(source)
FStringMiddle {
value: "=10",
is_raw: false,
triple_quoted: false,
},
5..8,
),
@ -36,6 +37,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
9..10,
),
@ -75,6 +77,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
19..20,
),
@ -124,6 +127,7 @@ expression: lex_source(source)
FStringMiddle {
value: " ",
is_raw: false,
triple_quoted: false,
},
31..32,
),

View file

@ -11,6 +11,7 @@ expression: lex_source(source)
FStringMiddle {
value: "\\0",
is_raw: false,
triple_quoted: false,
},
2..4,
),

View file

@ -54,6 +54,8 @@ pub enum Tok {
value: String,
/// Whether the string is raw or not.
is_raw: bool,
/// Whether the string is triple quoted.
triple_quoted: bool,
},
/// Token value for the end of an f-string. This includes the closing quote.
FStringEnd,