mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-12 07:35:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Rename AnyStringKind -> AnyStringFlags (#11405)

Browse source 
## Summary

This PR renames `AnyStringKind` to `AnyStringFlags` and `AnyStringFlags`
to `AnyStringFlagsInner`.

The main motivation is to have consistent usage of "kind" and "flags".
For each string kind, it's "flags" like `StringLiteralFlags`,
`BytesLiteralFlags`, and `FStringFlags` but it was `AnyStringKind` for
the "any" variant.
This commit is contained in:
Dhruv Manilawala 2024-05-13 18:48:07 +05:30 • committed by GitHub
parent be0ccabbaa
commit 6ecb4776de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
54 changed files with 378 additions and 371 deletions
Download patch file Download diff file Expand all files Collapse all files

20
crates/ruff_python_parser/src/lexer.rs
View file

@ -35,7 +35,7 @@ use unicode_ident::{is_xid_continue, is_xid_start};
use unicode_normalization::UnicodeNormalization;
use ruff_python_ast::{
str::Quote, AnyStringKind, AnyStringPrefix, FStringPrefix, Int, IpyEscapeKind,
str::Quote, AnyStringFlags, AnyStringPrefix, FStringPrefix, Int, IpyEscapeKind,
};
use ruff_text_size::{TextLen, TextRange, TextSize};
@ -561,7 +561,7 @@ impl<'source> Lexer<'source> {
#[cfg(debug_assertions)]
debug_assert_eq!(self.cursor.previous(), quote);
let mut kind = AnyStringKind::default()
let mut flags = AnyStringFlags::default()
.with_prefix(AnyStringPrefix::Format(prefix))
.with_quote_style(if quote == '"' {
Quote::Double
@ -570,11 +570,11 @@ impl<'source> Lexer<'source> {
});
if self.cursor.eat_char2(quote, quote) {
kind = kind.with_triple_quotes();
flags = flags.with_triple_quotes();
}
self.fstrings.push(FStringContext::new(kind, self.nesting));
Tok::FStringStart(kind)
self.fstrings.push(FStringContext::new(flags, self.nesting));
Tok::FStringStart(flags)
}
/// Lex a f-string middle or end token.
@ -707,7 +707,7 @@ impl<'source> Lexer<'source> {
};
Ok(Some(Tok::FStringMiddle {
value: value.into_boxed_str(),
kind: fstring.kind(),
flags: fstring.flags(),
}))
}
@ -716,7 +716,7 @@ impl<'source> Lexer<'source> {
#[cfg(debug_assertions)]
debug_assert_eq!(self.cursor.previous(), quote);
let mut kind = AnyStringKind::default()
let mut flags = AnyStringFlags::default()
.with_prefix(prefix)
.with_quote_style(if quote == '"' {
Quote::Double
@ -727,13 +727,13 @@ impl<'source> Lexer<'source> {
// If the next two characters are also the quote character, then we have a triple-quoted
// string; consume those two characters and ensure that we require a triple-quote to close
if self.cursor.eat_char2(quote, quote) {
kind = kind.with_triple_quotes();
flags = flags.with_triple_quotes();
}
let value_start = self.offset();
let quote_byte = u8::try_from(quote).expect("char that fits in u8");
let value_end = if kind.is_triple_quoted() {
let value_end = if flags.is_triple_quoted() {
// For triple-quoted strings, scan until we find the closing quote (ignoring escaped
// quotes) or the end of the file.
loop {
@ -821,7 +821,7 @@ impl<'source> Lexer<'source> {
value: self.source[TextRange::new(value_start, value_end)]
.to_string()
.into_boxed_str(),
kind,
flags,
})
}

24
crates/ruff_python_parser/src/lexer/fstring.rs
View file

@ -1,9 +1,9 @@
use ruff_python_ast::AnyStringKind;
use ruff_python_ast::AnyStringFlags;
/// The context representing the current f-string that the lexer is in.
#[derive(Debug)]
pub(crate) struct FStringContext {
kind: AnyStringKind,
flags: AnyStringFlags,
/// The level of nesting for the lexer when it entered the current f-string.
/// The nesting level includes all kinds of parentheses i.e., round, square,
@ -17,18 +17,18 @@ pub(crate) struct FStringContext {
}
impl FStringContext {
pub(crate) const fn new(kind: AnyStringKind, nesting: u32) -> Self {
debug_assert!(kind.is_f_string());
pub(crate) const fn new(flags: AnyStringFlags, nesting: u32) -> Self {
debug_assert!(flags.is_f_string());
Self {
kind,
flags,
nesting,
format_spec_depth: 0,
}
}
pub(crate) const fn kind(&self) -> AnyStringKind {
debug_assert!(self.kind.is_f_string());
self.kind
pub(crate) const fn flags(&self) -> AnyStringFlags {
debug_assert!(self.flags.is_f_string());
self.flags
}
pub(crate) const fn nesting(&self) -> u32 {
@ -37,14 +37,14 @@ impl FStringContext {
/// Returns the quote character for the current f-string.
pub(crate) const fn quote_char(&self) -> char {
self.kind.quote_style().as_char()
self.flags.quote_style().as_char()
}
/// Returns the triple quotes for the current f-string if it is a triple-quoted
/// f-string, `None` otherwise.
pub(crate) const fn triple_quotes(&self) -> Option<&'static str> {
if self.is_triple_quoted() {
Some(self.kind.quote_str())
Some(self.flags.quote_str())
} else {
None
}
@ -52,12 +52,12 @@ impl FStringContext {
/// Returns `true` if the current f-string is a raw f-string.
pub(crate) fn is_raw_string(&self) -> bool {
self.kind.is_raw_string()
self.flags.is_raw_string()
}
/// Returns `true` if the current f-string is a triple-quoted f-string.
pub(crate) const fn is_triple_quoted(&self) -> bool {
self.kind.is_triple_quoted()
self.flags.is_triple_quoted()
}
/// Calculates the number of open parentheses for the current f-string

14
crates/ruff_python_parser/src/parser/expression.rs
View file

@ -1231,17 +1231,17 @@ impl<'src> Parser<'src> {
///
/// See: <https://docs.python.org/3.13/reference/lexical_analysis.html#string-and-bytes-literals>
fn parse_string_or_byte_literal(&mut self) -> StringType {
let (Tok::String { value, kind }, range) = self.bump(TokenKind::String) else {
let (Tok::String { value, flags }, range) = self.bump(TokenKind::String) else {
unreachable!()
};
match parse_string_literal(value, kind, range) {
match parse_string_literal(value, flags, range) {
Ok(string) => string,
Err(error) => {
let location = error.location();
self.add_error(ParseErrorType::Lexical(error.into_error()), location);
if kind.is_byte_string() {
if flags.is_byte_string() {
// test_err invalid_byte_literal
// b'123a𝐁c'
// rb"a𝐁c123"
@ -1249,7 +1249,7 @@ impl<'src> Parser<'src> {
StringType::Bytes(ast::BytesLiteral {
value: Box::new([]),
range,
flags: ast::BytesLiteralFlags::from(kind).with_invalid(),
flags: ast::BytesLiteralFlags::from(flags).with_invalid(),
})
} else {
// test_err invalid_string_literal
@ -1258,7 +1258,7 @@ impl<'src> Parser<'src> {
StringType::Str(ast::StringLiteral {
value: "".into(),
range,
flags: ast::StringLiteralFlags::from(kind).with_invalid(),
flags: ast::StringLiteralFlags::from(flags).with_invalid(),
})
}
}
@ -1306,12 +1306,12 @@ impl<'src> Parser<'src> {
FStringElement::Expression(parser.parse_fstring_expression_element())
}
TokenKind::FStringMiddle => {
let (Tok::FStringMiddle { value, kind, .. }, range) = parser.next_token()
let (Tok::FStringMiddle { value, flags, .. }, range) = parser.next_token()
else {
unreachable!()
};
FStringElement::Literal(
parse_fstring_literal_element(value, kind, range).unwrap_or_else(
parse_fstring_literal_element(value, flags, range).unwrap_or_else(
|lex_error| {
// test_err invalid_fstring_literal_element
// f'hello \N{INVALID} world'

14
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__empty_fstrings.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -22,7 +22,7 @@ expression: lex_source(source)
(
String {
value: "",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -34,7 +34,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -50,7 +50,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -67,7 +67,7 @@ expression: lex_source(source)
(
String {
value: "",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -79,7 +79,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -95,7 +95,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__escape_unicode_name.snap
View file

@ -6,7 +6,7 @@ expression: lex_source(source)
(
String {
value: "\\N{EN SPACE}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

10
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "normal ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -45,7 +45,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " {another} ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -72,7 +72,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " {",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -99,7 +99,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

6
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\n# not a comment ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -59,7 +59,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " # not a comment\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

10
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_conversion.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -42,7 +42,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -83,7 +83,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -110,7 +110,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: ".3f!r",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -127,7 +127,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " {x!r}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

8
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -45,7 +45,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\\"\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -76,7 +76,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " \\\"\\\"\\\n end",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

16
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape_braces.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -48,7 +48,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -61,7 +61,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -91,7 +91,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -104,7 +104,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\{foo}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -120,7 +120,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -133,7 +133,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\\\{foo}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

8
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape_raw.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -20,7 +20,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -49,7 +49,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\\"\\",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -82,7 +82,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " \\\"\\\"\\\n end",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,

6
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_expression_multiline.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "first ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -71,7 +71,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " second",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

18
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_multiline.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\nhello\n world\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -34,7 +34,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -47,7 +47,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\n world\nhello\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -63,7 +63,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -76,7 +76,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "some ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -92,7 +92,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -105,7 +105,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "multiline\nallowed ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -140,7 +140,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " string",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_named_unicode.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\N{BULLET} normal \\Nope \\N",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

6
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_named_unicode_raw.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -20,7 +20,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\N",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -49,7 +49,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " normal",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,

26
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_nested.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "foo ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -34,7 +34,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -47,7 +47,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "bar ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -73,7 +73,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -116,7 +116,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " baz",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -132,7 +132,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -145,7 +145,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "foo ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -161,7 +161,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -174,7 +174,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "bar",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -195,7 +195,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " some ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -211,7 +211,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -224,7 +224,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "another",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

28
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_parentheses.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -29,7 +29,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -42,7 +42,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "{}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -58,7 +58,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -71,7 +71,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -95,7 +95,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -108,7 +108,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "{",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -129,7 +129,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -145,7 +145,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -158,7 +158,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "{{}}",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -174,7 +174,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -187,7 +187,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -208,7 +208,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " {} {",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -229,7 +229,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "} {{}} ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

20
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_prefix.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -21,7 +21,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -37,7 +37,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -55,7 +55,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -73,7 +73,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: true,
@ -91,7 +91,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: true,
@ -109,7 +109,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -127,7 +127,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: false,
@ -145,7 +145,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: true,
@ -163,7 +163,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Raw {
uppercase_r: true,

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_single_quote_escape_mac_eol.snap
View file

@ -5,7 +5,7 @@ expression: fstring_single_quote_escape_eol(MAC_EOL)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: fstring_single_quote_escape_eol(MAC_EOL)
(
FStringMiddle {
value: "text \\\r more text",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_single_quote_escape_unix_eol.snap
View file

@ -5,7 +5,7 @@ expression: fstring_single_quote_escape_eol(UNIX_EOL)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: fstring_single_quote_escape_eol(UNIX_EOL)
(
FStringMiddle {
value: "text \\\n more text",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_single_quote_escape_windows_eol.snap
View file

@ -5,7 +5,7 @@ expression: fstring_single_quote_escape_eol(WINDOWS_EOL)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: fstring_single_quote_escape_eol(WINDOWS_EOL)
(
FStringMiddle {
value: "text \\\r\n more text",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

20
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_format_spec.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -36,7 +36,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -77,7 +77,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: ".3f",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -94,7 +94,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -121,7 +121,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: ".",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -148,7 +148,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "f",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -165,7 +165,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -182,7 +182,7 @@ expression: lex_source(source)
(
String {
value: "",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -199,7 +199,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "*^",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -248,7 +248,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

6
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_ipy_escape_command.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "foo ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -49,7 +49,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " bar",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_lambda_expression.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -61,7 +61,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),

32
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_multiline_format_spec.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -49,7 +49,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "d\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -66,7 +66,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -86,7 +86,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -99,7 +99,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -130,7 +130,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "a\n b\n c\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -147,7 +147,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -167,7 +167,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -180,7 +180,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -211,7 +211,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "d",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -232,7 +232,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -252,7 +252,7 @@ expression: lex_source(source)
),
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -265,7 +265,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -296,7 +296,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "a",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -327,7 +327,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "__",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

10
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_named_expression.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -32,7 +32,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "=10",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -49,7 +49,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -94,7 +94,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),
@ -149,7 +149,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: " ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_nul_char.snap
View file

@ -5,7 +5,7 @@ expression: lex_source(source)
[
(
FStringStart(
StringKind {
AnyStringFlags {
prefix: Format(
Regular,
),
@ -18,7 +18,7 @@ expression: lex_source(source)
(
FStringMiddle {
value: "\\0",
kind: StringKind {
flags: AnyStringFlags {
prefix: Format(
Regular,
),

8
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__non_logical_newline_in_string_continuation.snap
View file

@ -14,7 +14,7 @@ expression: lex_source(source)
(
String {
value: "a",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -31,7 +31,7 @@ expression: lex_source(source)
(
String {
value: "b",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -52,7 +52,7 @@ expression: lex_source(source)
(
String {
value: "c",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -65,7 +65,7 @@ expression: lex_source(source)
(
String {
value: "d",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

18
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string.snap
View file

@ -6,7 +6,7 @@ expression: lex_source(source)
(
String {
value: "double",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -19,7 +19,7 @@ expression: lex_source(source)
(
String {
value: "single",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -32,7 +32,7 @@ expression: lex_source(source)
(
String {
value: "can\\'t",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -45,7 +45,7 @@ expression: lex_source(source)
(
String {
value: "\\\\\\\"",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -58,7 +58,7 @@ expression: lex_source(source)
(
String {
value: "\\t\\r\\n",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -71,7 +71,7 @@ expression: lex_source(source)
(
String {
value: "\\g",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -84,7 +84,7 @@ expression: lex_source(source)
(
String {
value: "raw\\'",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Raw {
uppercase: false,
@ -99,7 +99,7 @@ expression: lex_source(source)
(
String {
value: "\\420",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),
@ -112,7 +112,7 @@ expression: lex_source(source)
(
String {
value: "\\200\\0a",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_mac_eol.snap
View file

@ -6,7 +6,7 @@ expression: string_continuation_with_eol(MAC_EOL)
(
String {
value: "abc\\\rdef",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_unix_eol.snap
View file

@ -6,7 +6,7 @@ expression: string_continuation_with_eol(UNIX_EOL)
(
String {
value: "abc\\\ndef",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_windows_eol.snap
View file

@ -6,7 +6,7 @@ expression: string_continuation_with_eol(WINDOWS_EOL)
(
String {
value: "abc\\\r\ndef",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_mac_eol.snap
View file

@ -6,7 +6,7 @@ expression: triple_quoted_eol(MAC_EOL)
(
String {
value: "\r test string\r ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_unix_eol.snap
View file

@ -6,7 +6,7 @@ expression: triple_quoted_eol(UNIX_EOL)
(
String {
value: "\n test string\n ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_windows_eol.snap
View file

@ -6,7 +6,7 @@ expression: triple_quoted_eol(WINDOWS_EOL)
(
String {
value: "\r\n test string\r\n ",
kind: StringKind {
flags: AnyStringFlags {
prefix: Regular(
Empty,
),

44
crates/ruff_python_parser/src/string.rs
View file

@ -2,7 +2,7 @@
use bstr::ByteSlice;
use ruff_python_ast::{self as ast, AnyStringKind, Expr};
use ruff_python_ast::{self as ast, AnyStringFlags, Expr};
use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::lexer::{LexicalError, LexicalErrorType};
@ -44,8 +44,8 @@ struct StringParser {
source: Box<str>,
/// Current position of the parser in the source.
cursor: usize,
/// The kind of string.
kind: AnyStringKind,
/// Flags that can be used to query information about the string.
flags: AnyStringFlags,
/// The location of the first character in the source from the start of the file.
offset: TextSize,
/// The range of the string literal.
@ -53,11 +53,11 @@ struct StringParser {
}
impl StringParser {
fn new(source: Box<str>, kind: AnyStringKind, offset: TextSize, range: TextRange) -> Self {
fn new(source: Box<str>, flags: AnyStringFlags, offset: TextSize, range: TextRange) -> Self {
Self {
source,
cursor: 0,
kind,
flags,
offset,
range,
}
@ -214,9 +214,9 @@ impl StringParser {
'v' => '\x0b',
o @ '0'..='7' => self.parse_octet(o as u8),
'x' => self.parse_unicode_literal(2)?,
'u' if !self.kind.is_byte_string() => self.parse_unicode_literal(4)?,
'U' if !self.kind.is_byte_string() => self.parse_unicode_literal(8)?,
'N' if !self.kind.is_byte_string() => self.parse_unicode_name()?,
'u' if !self.flags.is_byte_string() => self.parse_unicode_literal(4)?,
'U' if !self.flags.is_byte_string() => self.parse_unicode_literal(8)?,
'N' if !self.flags.is_byte_string() => self.parse_unicode_name()?,
// Special cases where the escape sequence is not a single character
'\n' => return Ok(None),
'\r' => {
@ -282,7 +282,7 @@ impl StringParser {
// raise a syntax error as is done by the CPython parser. It might
// be supported in the future, refer to point 3: https://peps.python.org/pep-0701/#rejected-ideas
b'\\' => {
if !self.kind.is_raw_string() && self.peek_byte().is_some() {
if !self.flags.is_raw_string() && self.peek_byte().is_some() {
match self.parse_escaped_char()? {
None => {}
Some(EscapedChar::Literal(c)) => value.push(c),
@ -330,12 +330,12 @@ impl StringParser {
));
}
if self.kind.is_raw_string() {
if self.flags.is_raw_string() {
// For raw strings, no escaping is necessary.
return Ok(StringType::Bytes(ast::BytesLiteral {
value: self.source.into_boxed_bytes(),
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}));
}
@ -344,7 +344,7 @@ impl StringParser {
return Ok(StringType::Bytes(ast::BytesLiteral {
value: self.source.into_boxed_bytes(),
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}));
};
@ -381,17 +381,17 @@ impl StringParser {
Ok(StringType::Bytes(ast::BytesLiteral {
value: value.into_boxed_slice(),
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}))
}
fn parse_string(mut self) -> Result<StringType, LexicalError> {
if self.kind.is_raw_string() {
if self.flags.is_raw_string() {
// For raw strings, no escaping is necessary.
return Ok(StringType::Str(ast::StringLiteral {
value: self.source,
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}));
}
@ -400,7 +400,7 @@ impl StringParser {
return Ok(StringType::Str(ast::StringLiteral {
value: self.source,
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}));
};
@ -437,12 +437,12 @@ impl StringParser {
Ok(StringType::Str(ast::StringLiteral {
value: value.into_boxed_str(),
range: self.range,
flags: self.kind.into(),
flags: self.flags.into(),
}))
}
fn parse(self) -> Result<StringType, LexicalError> {
if self.kind.is_byte_string() {
if self.flags.is_byte_string() {
self.parse_bytes()
} else {
self.parse_string()
@ -452,19 +452,19 @@ impl StringParser {
pub(crate) fn parse_string_literal(
source: Box<str>,
kind: AnyStringKind,
flags: AnyStringFlags,
range: TextRange,
) -> Result<StringType, LexicalError> {
StringParser::new(source, kind, range.start() + kind.opener_len(), range).parse()
StringParser::new(source, flags, range.start() + flags.opener_len(), range).parse()
}
// TODO(dhruvmanila): Move this to the new parser
pub(crate) fn parse_fstring_literal_element(
source: Box<str>,
kind: AnyStringKind,
flags: AnyStringFlags,
range: TextRange,
) -> Result<ast::FStringLiteralElement, LexicalError> {
StringParser::new(source, kind, range.start(), range).parse_fstring_middle()
StringParser::new(source, flags, range.start(), range).parse_fstring_middle()
}
#[cfg(test)]

12
crates/ruff_python_parser/src/token.rs
View file

@ -5,7 +5,7 @@
//!
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens
use ruff_python_ast::{AnyStringKind, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
use std::fmt;
use crate::Mode;
@ -44,11 +44,11 @@ pub enum Tok {
value: Box<str>,
/// Flags that can be queried to determine the quote style
/// and prefixes of the string
kind: AnyStringKind,
flags: AnyStringFlags,
},
/// Token value for the start of an f-string. This includes the `f`/`F`/`fr` prefix
/// and the opening quote(s).
FStringStart(AnyStringKind),
FStringStart(AnyStringFlags),
/// Token value that includes the portion of text inside the f-string that's not
/// part of the expression part and isn't an opening or closing brace.
FStringMiddle {
@ -56,7 +56,7 @@ pub enum Tok {
value: Box<str>,
/// Flags that can be queried to determine the quote style
/// and prefixes of the string
kind: AnyStringKind,
flags: AnyStringFlags,
},
/// Token value for the end of an f-string. This includes the closing quote.
FStringEnd,
@ -245,8 +245,8 @@ impl fmt::Display for Tok {
Int { value } => write!(f, "{value}"),
Float { value } => write!(f, "{value}"),
Complex { real, imag } => write!(f, "{real}j{imag}"),
String { value, kind } => {
write!(f, "{}", kind.format_string_contents(value))
String { value, flags } => {
write!(f, "{}", flags.format_string_contents(value))
}
FStringStart(_) => f.write_str("FStringStart"),
FStringMiddle { value, .. } => f.write_str(value),