Start tracking quoting style in the AST (#10298)

This PR modifies our AST so that nodes for string literals, bytes literals and f-strings all retain the following information:
- The quoting style used (double or single quotes)
- Whether the string is triple-quoted or not
- Whether the string is raw or not

This PR is a followup to #10256. Like with that PR, this PR does not, in itself, fix any bugs. However, it means that we will have the necessary information to preserve quoting style and rawness of strings in the `ExprGenerator` in a followup PR, which will allow us to provide a fix for https://github.com/astral-sh/ruff/issues/7799.

The information is recorded on the AST nodes using a bitflag field on each node, similarly to how we recorded the information on `Tok::String`, `Tok::FStringStart` and `Tok::FStringMiddle` tokens in #10298. Rather than reusing the bitflag I used for the tokens, however, I decided to create a custom bitflag for each AST node.

Using different bitflags for each node allows us to make invalid states unrepresentable: it is valid to set a `u` prefix on a string literal, but not on a bytes literal or an f-string. It also allows us to have better debug representations for each AST node modified in this PR.
This commit is contained in:
Alex Waygood 2024-03-08 19:11:47 +00:00 committed by GitHub
parent 965adbed4b
commit 1d97f27335
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
81 changed files with 4733 additions and 3756 deletions

View file

@ -115,7 +115,7 @@ pub use parser::{
};
use ruff_python_ast::{Mod, PySourceType, Suite};
pub use string::FStringErrorType;
pub use string_token_flags::{QuoteStyle, StringKind};
pub use string_token_flags::StringKind;
pub use token::{Tok, TokenKind};
use crate::lexer::LexResult;

View file

@ -1626,10 +1626,11 @@ StringLiteral: StringType = {
};
FStringExpr: StringType = {
<location:@L> FStringStart <elements:FStringMiddlePattern*> FStringEnd <end_location:@R> => {
<location:@L> <start:fstring_start> <elements:FStringMiddlePattern*> FStringEnd <end_location:@R> => {
StringType::FString(ast::FString {
elements,
range: (location..end_location).into()
range: (location..end_location).into(),
flags: start.into()
})
}
};
@ -2002,7 +2003,7 @@ extern {
Dedent => token::Tok::Dedent,
StartModule => token::Tok::StartModule,
StartExpression => token::Tok::StartExpression,
FStringStart => token::Tok::FStringStart(StringKind),
fstring_start => token::Tok::FStringStart(<StringKind>),
FStringEnd => token::Tok::FStringEnd,
"!" => token::Tok::Exclamation,
"?" => token::Tok::Question,

File diff suppressed because it is too large Load diff

View file

@ -19,7 +19,11 @@ Ok(
StringLiteral {
range: 0..5,
value: "foo",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -15,7 +15,11 @@ Dict(
StringLiteral {
range: 1..4,
value: "a",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -32,7 +36,11 @@ Dict(
StringLiteral {
range: 16..19,
value: "d",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -49,7 +57,11 @@ Dict(
StringLiteral {
range: 6..9,
value: "b",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -70,7 +82,11 @@ Dict(
StringLiteral {
range: 21..24,
value: "e",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -26,7 +26,11 @@ expression: parse_ast
StringLiteral {
range: 3..7,
value: " f",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -38,6 +42,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -74,6 +83,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -120,6 +134,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -180,6 +199,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -228,7 +252,11 @@ expression: parse_ast
StringLiteral {
range: 46..49,
value: "}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -251,6 +279,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),
@ -299,7 +332,11 @@ expression: parse_ast
StringLiteral {
range: 62..65,
value: "{",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -322,6 +359,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),
@ -363,6 +405,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -416,6 +463,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -457,6 +509,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -516,6 +573,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -582,6 +644,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -606,6 +673,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),
@ -632,7 +704,11 @@ expression: parse_ast
StringLiteral {
range: 173..179,
value: "foo ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -680,13 +756,22 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 196..201,
value: "baz",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
],
@ -734,7 +819,11 @@ expression: parse_ast
StringLiteral {
range: 227..232,
value: "one",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -766,12 +855,20 @@ expression: parse_ast
StringLiteral {
range: 256..269,
value: "implicitly ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
StringLiteral {
range: 270..284,
value: "concatenated",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
],
value: "implicitly concatenated",
@ -861,6 +958,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -888,6 +990,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),
@ -936,6 +1043,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: true,
},
},
),
),
@ -977,6 +1089,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..6,
value: "foo",
unicode: true,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "u",
triple_quoted: false,
},
},
),
FString(
@ -39,20 +43,33 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 16..21,
value: "baz",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 22..29,
value: " some",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
],
@ -75,7 +92,11 @@ expression: parse_ast
StringLiteral {
range: 30..35,
value: "foo",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -98,20 +119,33 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 45..51,
value: "baz",
unicode: true,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "u",
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 52..59,
value: " some",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
],
@ -134,7 +168,11 @@ expression: parse_ast
StringLiteral {
range: 60..65,
value: "foo",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -157,20 +195,33 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 75..80,
value: "baz",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 81..89,
value: " some",
unicode: true,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "u",
triple_quoted: false,
},
},
),
],
@ -193,7 +244,11 @@ expression: parse_ast
StringLiteral {
range: 90..96,
value: "foo",
unicode: true,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "u",
triple_quoted: false,
},
},
),
FString(
@ -228,20 +283,33 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 117..123,
value: "bar",
unicode: true,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "u",
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 124..128,
value: "no",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
],

View file

@ -16,7 +16,11 @@ Call(
StringLiteral {
range: 0..3,
value: " ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -77,7 +81,11 @@ Call(
StringLiteral {
range: 43..53,
value: "LIMIT %d",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -121,7 +129,11 @@ Call(
StringLiteral {
range: 91..102,
value: "OFFSET %d",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -19,7 +19,11 @@ expression: parse_ast
StringLiteral {
range: 8..14,
value: "test",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -108,7 +112,11 @@ expression: parse_ast
StringLiteral {
range: 81..88,
value: "label",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -125,7 +133,11 @@ expression: parse_ast
StringLiteral {
range: 90..96,
value: "test",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -149,7 +161,11 @@ expression: parse_ast
StringLiteral {
range: 118..125,
value: "label",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -1,6 +1,6 @@
---
source: crates/ruff_python_parser/src/parser.rs
expression: "parse_suite(source, \"<test>\").unwrap()"
expression: parse_suite(source).unwrap()
---
[
ClassDef(
@ -121,7 +121,11 @@ expression: "parse_suite(source, \"<test>\").unwrap()"
StringLiteral {
range: 80..89,
value: "default",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -22,6 +22,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -27,7 +27,11 @@ expression: parse_ast
StringLiteral {
range: 8..20,
value: "positional",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -27,7 +27,11 @@ expression: parse_ast
StringLiteral {
range: 6..19,
value: "Hello world",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -27,7 +27,11 @@ expression: parse_ast
StringLiteral {
range: 6..19,
value: "Hello world",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..13,
value: "Hello world",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -86,7 +86,11 @@ expression: parse_suite(source).unwrap()
StringLiteral {
range: 48..61,
value: "ForwardRefY",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -506,7 +506,11 @@ expression: parse_ast
StringLiteral {
range: 484..489,
value: "seq",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -541,7 +545,11 @@ expression: parse_ast
StringLiteral {
range: 518..523,
value: "map",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -838,7 +846,11 @@ expression: parse_ast
StringLiteral {
range: 664..667,
value: "X",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -1574,7 +1586,11 @@ expression: parse_ast
StringLiteral {
range: 1287..1292,
value: "foo",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -2500,7 +2516,11 @@ expression: parse_ast
StringLiteral {
range: 2036..2038,
value: "",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -2550,7 +2570,11 @@ expression: parse_ast
StringLiteral {
range: 2064..2066,
value: "",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -3174,7 +3198,11 @@ expression: parse_ast
StringLiteral {
range: 2449..2452,
value: "X",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -127,6 +127,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),
@ -227,6 +232,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -32,7 +32,11 @@ expression: parse_ast
StringLiteral {
range: 30..34,
value: "eg",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -276,6 +280,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),
@ -407,6 +416,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -23,7 +23,11 @@ expression: parse_ast
StringLiteral {
range: 4..37,
value: "\u{8}another cool trick",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..15,
value: "\u{8}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..9,
value: "\u{7}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..21,
value: "\r",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..45,
value: "\u{89}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..12,
value: "\u{7f}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -23,7 +23,11 @@ expression: parse_ast
StringLiteral {
range: 7..16,
value: "\u{3}8[1m",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -271,6 +271,11 @@ expression: parse_ast
254,
255,
],
flags: BytesLiteralFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..12,
value: "\u{1b}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -25,6 +25,11 @@ expression: parse_ast
97,
97,
],
flags: BytesLiteralFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
},

View file

@ -20,6 +20,11 @@ expression: parse_ast
83,
52,
],
flags: BytesLiteralFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..15,
value: "\u{c}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -64,6 +64,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -37,6 +37,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -37,6 +37,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -37,6 +37,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: true,
triple_quoted: false,
},
},
),
),

View file

@ -36,6 +36,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -68,6 +68,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -48,6 +48,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -37,6 +37,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: true,
},
},
),
),

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..9,
value: "\u{88}",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -1,6 +1,6 @@
---
source: crates/ruff_python_parser/src/string.rs
expression: "parse_suite(r#\"f\"\"\"#, \"<test>\").unwrap()"
expression: "parse_suite(r#\"f\"\"\"#).unwrap()"
---
[
Expr(
@ -15,6 +15,11 @@ expression: "parse_suite(r#\"f\"\"\"#, \"<test>\").unwrap()"
FString {
range: 0..3,
elements: [],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -30,6 +34,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
],

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -30,6 +34,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
],

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -40,7 +44,11 @@ expression: parse_ast
StringLiteral {
range: 17..20,
value: "!",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -52,6 +60,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
],

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
FString(
@ -40,7 +44,11 @@ expression: parse_ast
StringLiteral {
range: 17..20,
value: "!",
unicode: false,
flags: StringLiteralFlags {
quote_style: Double,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -52,13 +60,22 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 23..31,
value: "again!",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
],

View file

@ -52,6 +52,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -50,6 +50,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -44,12 +44,20 @@ expression: parse_ast
StringLiteral {
range: 8..10,
value: "",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
StringLiteral {
range: 11..13,
value: "",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
],
value: "",
@ -69,6 +77,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -52,6 +52,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -42,7 +42,11 @@ expression: parse_ast
StringLiteral {
range: 8..10,
value: "",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},
@ -59,6 +63,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -50,6 +50,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -43,6 +43,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -36,6 +36,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -36,6 +36,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -30,6 +30,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: false,
triple_quoted: false,
},
},
),
),

View file

@ -16,12 +16,20 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
StringLiteral {
range: 9..16,
value: "world",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
],
value: "Hello world",

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..20,
value: "Hello, world!",
unicode: true,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "u",
triple_quoted: true,
},
},
),
},

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..9,
value: "Hello ",
unicode: true,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "u",
triple_quoted: false,
},
},
),
FString(
@ -30,6 +34,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
],

View file

@ -16,7 +16,11 @@ expression: parse_ast
StringLiteral {
range: 0..9,
value: "Hello ",
unicode: true,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "u",
triple_quoted: false,
},
},
),
FString(
@ -30,13 +34,22 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
Literal(
StringLiteral {
range: 19..22,
value: "!",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
],

View file

@ -16,12 +16,20 @@ expression: parse_ast
StringLiteral {
range: 0..8,
value: "Hello ",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
StringLiteral {
range: 9..17,
value: "world",
unicode: true,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "u",
triple_quoted: false,
},
},
],
value: "Hello world",

View file

@ -16,12 +16,20 @@ expression: parse_ast
StringLiteral {
range: 0..9,
value: "Hello ",
unicode: true,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "u",
triple_quoted: false,
},
},
StringLiteral {
range: 10..17,
value: "world",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
],
value: "Hello world",

View file

@ -19,6 +19,11 @@ expression: parse_ast
49,
122,
],
flags: BytesLiteralFlags {
quote_style: Single,
raw: true,
triple_quoted: false,
},
},
),
},

View file

@ -17,6 +17,11 @@ expression: parse_ast
92,
92,
],
flags: BytesLiteralFlags {
quote_style: Single,
raw: true,
triple_quoted: false,
},
},
),
},

View file

@ -31,6 +31,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: true,
triple_quoted: false,
},
},
),
),

View file

@ -271,6 +271,11 @@ expression: parse_ast
254,
255,
],
flags: BytesLiteralFlags {
quote_style: Single,
raw: false,
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..18,
value: "text more text",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..18,
value: "text more text",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -14,7 +14,11 @@ expression: parse_ast
StringLiteral {
range: 0..19,
value: "text more text",
unicode: false,
flags: StringLiteralFlags {
quote_style: Single,
prefix: "",
triple_quoted: false,
},
},
),
},

View file

@ -31,6 +31,11 @@ expression: parse_ast
},
),
],
flags: FStringFlags {
quote_style: Double,
raw: true,
triple_quoted: true,
},
},
),
),

View file

@ -308,6 +308,7 @@ impl StringParser {
return Ok(StringType::Bytes(ast::BytesLiteral {
value: self.source.into_boxed_bytes(),
range: self.range,
flags: self.kind.into(),
}));
}
@ -316,6 +317,7 @@ impl StringParser {
return Ok(StringType::Bytes(ast::BytesLiteral {
value: self.source.into_boxed_bytes(),
range: self.range,
flags: self.kind.into(),
}));
};
@ -352,6 +354,7 @@ impl StringParser {
Ok(StringType::Bytes(ast::BytesLiteral {
value: value.into_boxed_slice(),
range: self.range,
flags: self.kind.into(),
}))
}
@ -360,8 +363,8 @@ impl StringParser {
// For raw strings, no escaping is necessary.
return Ok(StringType::Str(ast::StringLiteral {
value: self.source,
unicode: self.kind.is_u_string(),
range: self.range,
flags: self.kind.into(),
}));
}
@ -369,8 +372,8 @@ impl StringParser {
// If the string doesn't contain any escape sequences, return the owned string.
return Ok(StringType::Str(ast::StringLiteral {
value: self.source,
unicode: self.kind.is_u_string(),
range: self.range,
flags: self.kind.into(),
}));
};
@ -406,8 +409,8 @@ impl StringParser {
Ok(StringType::Str(ast::StringLiteral {
value: value.into_boxed_str(),
unicode: self.kind.is_u_string(),
range: self.range,
flags: self.kind.into(),
}))
}

View file

@ -2,6 +2,7 @@ use std::fmt;
use bitflags::bitflags;
use ruff_python_ast::{str::QuoteStyle, StringLiteralPrefix};
use ruff_text_size::{TextLen, TextSize};
bitflags! {
@ -287,28 +288,67 @@ impl fmt::Debug for StringKind {
}
}
#[derive(Debug, Default, Copy, Clone, Hash, PartialEq, Eq)]
pub enum QuoteStyle {
/// E.g. '
Single,
/// E.g. "
#[default]
Double,
}
impl From<StringKind> for ruff_python_ast::StringLiteralFlags {
fn from(value: StringKind) -> ruff_python_ast::StringLiteralFlags {
debug_assert!(!value.is_f_string());
debug_assert!(!value.is_byte_string());
impl QuoteStyle {
pub const fn as_char(self) -> char {
match self {
Self::Single => '\'',
Self::Double => '"',
let mut new = ruff_python_ast::StringLiteralFlags::default();
if value.quote_style().is_double() {
new = new.with_double_quotes();
}
}
#[must_use]
pub const fn opposite(self) -> Self {
match self {
Self::Single => Self::Double,
Self::Double => Self::Single,
if value.is_triple_quoted() {
new = new.with_triple_quotes();
}
new.with_prefix({
if value.is_u_string() {
debug_assert!(!value.is_raw_string());
StringLiteralPrefix::UString
} else if value.is_raw_string() {
StringLiteralPrefix::RString
} else {
StringLiteralPrefix::None
}
})
}
}
impl From<StringKind> for ruff_python_ast::BytesLiteralFlags {
fn from(value: StringKind) -> ruff_python_ast::BytesLiteralFlags {
debug_assert!(value.is_byte_string());
debug_assert!(!value.is_f_string());
debug_assert!(!value.is_u_string());
let mut new = ruff_python_ast::BytesLiteralFlags::default();
if value.quote_style().is_double() {
new = new.with_double_quotes();
}
if value.is_triple_quoted() {
new = new.with_triple_quotes();
}
if value.is_raw_string() {
new = new.with_r_prefix();
}
new
}
}
impl From<StringKind> for ruff_python_ast::FStringFlags {
fn from(value: StringKind) -> ruff_python_ast::FStringFlags {
debug_assert!(value.is_f_string());
debug_assert!(!value.is_byte_string());
debug_assert!(!value.is_u_string());
let mut new = ruff_python_ast::FStringFlags::default();
if value.quote_style().is_double() {
new = new.with_double_quotes();
}
if value.is_triple_quoted() {
new = new.with_triple_quotes();
}
if value.is_raw_string() {
new = new.with_r_prefix();
}
new
}
}