language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 18:28:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add support for PEP 701 (#7376)

Browse source 
## Summary

This PR adds support for PEP 701 in Ruff. This is a rollup PR of all the
other individual PRs. The separate PRs were created for logic separation
and code reviews. Refer to each pull request for a detail description on
the change.

Refer to the PR description for the list of pull requests within this PR.

## Test Plan

### Formatter ecosystem checks

Explanation for the change in ecosystem check:
https://github.com/astral-sh/ruff/pull/7597#issue-1908878183

#### `main`

```
| project      | similarity index  | total files       | changed files     |
|--------------|------------------:|------------------:|------------------:|
| cpython      |           0.76083 |              1789 |              1631 |
| django       |           0.99983 |              2760 |                36 |
| transformers |           0.99963 |              2587 |               319 |
| twine        |           1.00000 |                33 |                 0 |
| typeshed     |           0.99983 |              3496 |                18 |
| warehouse    |           0.99967 |               648 |                15 |
| zulip        |           0.99972 |              1437 |                21 |
```

#### `dhruv/pep-701`

```
| project      | similarity index  | total files       | changed files     |
|--------------|------------------:|------------------:|------------------:|
| cpython      |           0.76051 |              1789 |              1632 |
| django       |           0.99983 |              2760 |                36 |
| transformers |           0.99963 |              2587 |               319 |
| twine        |           1.00000 |                33 |                 0 |
| typeshed     |           0.99983 |              3496 |                18 |
| warehouse    |           0.99967 |               648 |                15 |
| zulip        |           0.99972 |              1437 |                21 |
```
This commit is contained in:
Dhruv Manilawala 2023-09-29 08:25:39 +05:30 • committed by GitHub
parent 78b8741352
commit e62e245c61
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
115 changed files with 44780 additions and 31370 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/ruff_python_parser/Cargo.toml
View file

@ -18,6 +18,7 @@ ruff_python_ast = { path = "../ruff_python_ast" }
ruff_text_size = { path = "../ruff_text_size" }
anyhow = { workspace = true }
bitflags = { workspace = true }
is-macro = { workspace = true }
itertools = { workspace = true }
lalrpop-util = { version = "0.20.0", default-features = false }

422
crates/ruff_python_parser/src/lexer.rs
View file

@ -37,6 +37,7 @@ use ruff_python_ast::{Int, IpyEscapeKind};
use ruff_text_size::{TextLen, TextRange, TextSize};
use crate::lexer::cursor::{Cursor, EOF_CHAR};
use crate::lexer::fstring::{FStringContext, FStringContextFlags, FStrings};
use crate::lexer::indentation::{Indentation, Indentations};
use crate::{
soft_keywords::SoftKeywordTransformer,
@ -46,6 +47,7 @@ use crate::{
};
mod cursor;
mod fstring;
mod indentation;
/// A lexer for Python source code.
@ -62,6 +64,8 @@ pub struct Lexer<'source> {
pending_indentation: Option<Indentation>,
// Lexer mode.
mode: Mode,
// F-string contexts.
fstrings: FStrings,
}
/// Contains a Token along with its `range`.
@ -154,6 +158,7 @@ impl<'source> Lexer<'source> {
source: input,
cursor: Cursor::new(input),
mode,
fstrings: FStrings::default(),
};
// TODO: Handle possible mismatch between BOM and explicit encoding declaration.
// spell-checker:ignore feff
@ -165,16 +170,24 @@ impl<'source> Lexer<'source> {
/// Lex an identifier. Also used for keywords and string/bytes literals with a prefix.
fn lex_identifier(&mut self, first: char) -> Result<Tok, LexicalError> {
// Detect potential string like rb'' b'' f'' u'' r''
match self.cursor.first() {
quote @ ('\'' | '"') => {
match (first, self.cursor.first()) {
('f' | 'F', quote @ ('\'' | '"')) => {
self.cursor.bump();
return Ok(self.lex_fstring_start(quote, false));
}
('r' | 'R', 'f' | 'F') | ('f' | 'F', 'r' | 'R') if is_quote(self.cursor.second()) => {
self.cursor.bump();
let quote = self.cursor.bump().unwrap();
return Ok(self.lex_fstring_start(quote, true));
}
(_, quote @ ('\'' | '"')) => {
if let Ok(string_kind) = StringKind::try_from(first) {
self.cursor.bump();
return self.lex_string(string_kind, quote);
}
}
second @ ('f' | 'F' | 'r' | 'R' | 'b' | 'B') if is_quote(self.cursor.second()) => {
(_, second @ ('r' | 'R' | 'b' | 'B')) if is_quote(self.cursor.second()) => {
self.cursor.bump();
if let Ok(string_kind) = StringKind::try_from([first, second]) {
let quote = self.cursor.bump().unwrap();
return self.lex_string(string_kind, quote);
@ -509,6 +522,148 @@ impl<'source> Lexer<'source> {
}
}
/// Lex a f-string start token.
fn lex_fstring_start(&mut self, quote: char, is_raw_string: bool) -> Tok {
#[cfg(debug_assertions)]
debug_assert_eq!(self.cursor.previous(), quote);
let mut flags = FStringContextFlags::empty();
if quote == '"' {
flags |= FStringContextFlags::DOUBLE;
}
if is_raw_string {
flags |= FStringContextFlags::RAW;
}
if self.cursor.eat_char2(quote, quote) {
flags |= FStringContextFlags::TRIPLE;
}
self.fstrings.push(FStringContext::new(flags, self.nesting));
Tok::FStringStart
}
/// Lex a f-string middle or end token.
fn lex_fstring_middle_or_end(&mut self) -> Result<Option<Tok>, LexicalError> {
// SAFETY: Safe because the function is only called when `self.fstrings` is not empty.
let fstring = self.fstrings.current().unwrap();
self.cursor.start_token();
// Check if we're at the end of the f-string.
if fstring.is_triple_quoted() {
let quote_char = fstring.quote_char();
if self.cursor.eat_char3(quote_char, quote_char, quote_char) {
return Ok(Some(Tok::FStringEnd));
}
} else if self.cursor.eat_char(fstring.quote_char()) {
return Ok(Some(Tok::FStringEnd));
}
// We have to decode `{{` and `}}` into `{` and `}` respectively. As an
// optimization, we only allocate a new string we find any escaped curly braces,
// otherwise this string will remain empty and we'll use a source slice instead.
let mut normalized = String::new();
// Tracks the last offset of token value that has been written to `normalized`.
let mut last_offset = self.offset();
let mut in_named_unicode = false;
loop {
match self.cursor.first() {
// The condition is to differentiate between the `NUL` (`\0`) character
// in the source code and the one returned by `self.cursor.first()` when
// we reach the end of the source code.
EOF_CHAR if self.cursor.is_eof() => {
let error = if fstring.is_triple_quoted() {
FStringErrorType::UnterminatedTripleQuotedString
} else {
FStringErrorType::UnterminatedString
};
return Err(LexicalError {
error: LexicalErrorType::FStringError(error),
location: self.offset(),
});
}
'\n' if !fstring.is_triple_quoted() => {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
location: self.offset(),
});
}
'\\' => {
self.cursor.bump(); // '\'
if matches!(self.cursor.first(), '{' | '}') {
// Don't consume `{` or `}` as we want them to be emitted as tokens.
// They will be handled in the next iteration.
continue;
} else if !fstring.is_raw_string() {
if self.cursor.eat_char2('N', '{') {
in_named_unicode = true;
continue;
}
}
// Consume the escaped character.
self.cursor.bump();
}
quote @ ('\'' | '"') if quote == fstring.quote_char() => {
if let Some(triple_quotes) = fstring.triple_quotes() {
if self.cursor.rest().starts_with(triple_quotes) {
break;
}
self.cursor.bump();
} else {
break;
}
}
'{' => {
if self.cursor.second() == '{' {
self.cursor.bump();
normalized
.push_str(&self.source[TextRange::new(last_offset, self.offset())]);
self.cursor.bump(); // Skip the second `{`
last_offset = self.offset();
} else {
break;
}
}
'}' => {
if in_named_unicode {
in_named_unicode = false;
self.cursor.bump();
} else if self.cursor.second() == '}'
&& !fstring.is_in_format_spec(self.nesting)
{
self.cursor.bump();
normalized
.push_str(&self.source[TextRange::new(last_offset, self.offset())]);
self.cursor.bump(); // Skip the second `}`
last_offset = self.offset();
} else {
break;
}
}
_ => {
self.cursor.bump();
}
}
}
let range = self.token_range();
if range.is_empty() {
return Ok(None);
}
let value = if normalized.is_empty() {
self.source[range].to_string()
} else {
normalized.push_str(&self.source[TextRange::new(last_offset, self.offset())]);
normalized
};
Ok(Some(Tok::FStringMiddle {
value,
is_raw: fstring.is_raw_string(),
}))
}
/// Lex a string literal.
fn lex_string(&mut self, kind: StringKind, quote: char) -> Result<Tok, LexicalError> {
#[cfg(debug_assertions)]
@ -530,6 +685,19 @@ impl<'source> Lexer<'source> {
}
}
Some('\r' | '\n') if !triple_quoted => {
if let Some(fstring) = self.fstrings.current() {
// When we are in an f-string, check whether does the initial quote
// matches with f-strings quotes and if it is, then this must be a
// missing '}' token so raise the proper error.
if fstring.quote_char() == quote && !fstring.is_triple_quoted() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace,
),
location: self.offset() - fstring.quote_size(),
});
}
}
return Err(LexicalError {
error: LexicalErrorType::OtherError(
"EOL while scanning string literal".to_owned(),
@ -549,6 +717,21 @@ impl<'source> Lexer<'source> {
Some(_) => {}
None => {
if let Some(fstring) = self.fstrings.current() {
// When we are in an f-string, check whether does the initial quote
// matches with f-strings quotes and if it is, then this must be a
// missing '}' token so raise the proper error.
if fstring.quote_char() == quote
&& fstring.is_triple_quoted() == triple_quoted
{
return Err(LexicalError {
error: LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace,
),
location: self.offset() - fstring.quote_size(),
});
}
}
return Err(LexicalError {
error: if triple_quoted {
LexicalErrorType::Eof
@ -572,8 +755,28 @@ impl<'source> Lexer<'source> {
// This is the main entry point. Call this function to retrieve the next token.
// This function is used by the iterator implementation.
pub fn next_token(&mut self) -> LexResult {
if let Some(fstring) = self.fstrings.current() {
if !fstring.is_in_expression(self.nesting) {
match self.lex_fstring_middle_or_end() {
Ok(Some(tok)) => {
if tok == Tok::FStringEnd {
self.fstrings.pop();
}
return Ok((tok, self.token_range()));
}
Err(e) => {
// This is to prevent an infinite loop in which the lexer
// continuously returns an error token because the f-string
// remains on the stack.
self.fstrings.pop();
return Err(e);
}
_ => {}
}
}
}
// Return dedent tokens until the current indentation level matches the indentation of the next token.
if let Some(indentation) = self.pending_indentation.take() {
else if let Some(indentation) = self.pending_indentation.take() {
match self.indentations.current().try_compare(indentation) {
Ok(Ordering::Greater) => {
self.pending_indentation = Some(indentation);
@ -894,10 +1097,7 @@ impl<'source> Lexer<'source> {
if self.cursor.eat_char('=') {
Tok::NotEqual
} else {
return Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: '!' },
location: self.token_start(),
});
Tok::Exclamation
}
}
'~' => Tok::Tilde,
@ -922,11 +1122,26 @@ impl<'source> Lexer<'source> {
Tok::Lbrace
}
'}' => {
if let Some(fstring) = self.fstrings.current_mut() {
if fstring.nesting() == self.nesting {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::SingleRbrace),
location: self.token_start(),
});
}
fstring.try_end_format_spec(self.nesting);
}
self.nesting = self.nesting.saturating_sub(1);
Tok::Rbrace
}
':' => {
if self.cursor.eat_char('=') {
if self
.fstrings
.current_mut()
.is_some_and(|fstring| fstring.try_start_format_spec(self.nesting))
{
Tok::Colon
} else if self.cursor.eat_char('=') {
Tok::ColonEqual
} else {
Tok::Colon
@ -1743,4 +1958,191 @@ def f(arg=%timeit a = b):
.collect();
assert_debug_snapshot!(tokens);
}
#[test]
fn test_empty_fstrings() {
let source = r#"f"" "" F"" f'' '' f"""""" f''''''"#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_prefix() {
let source = r#"f"" F"" rf"" rF"" Rf"" RF"" fr"" Fr"" fR"" FR"""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring() {
let source = r#"f"normal {foo} {{another}} {bar} {{{three}}}""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_parentheses() {
let source = r#"f"{}" f"{{}}" f" {}" f"{{{}}}" f"{{{{}}}}" f" {} {{}} {{{}}} {{{{}}}} ""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_escape() {
let source = r#"f"\{x:\"\{x}} \"\"\
end""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_escape_braces() {
let source = r"f'\{foo}' f'\\{foo}' f'\{{foo}}' f'\\{{foo}}'";
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_escape_raw() {
let source = r#"rf"\{x:\"\{x}} \"\"\
end""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_named_unicode() {
let source = r#"f"\N{BULLET} normal \Nope \N""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_named_unicode_raw() {
let source = r#"rf"\N{BULLET} normal""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_with_named_expression() {
let source = r#"f"{x:=10} {(x:=10)} {x,{y:=10}} {[x:=10]}""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_with_format_spec() {
let source = r#"f"{foo:} {x=!s:.3f} {x:.{y}f} {'':*^{1:{1}}}""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_conversion() {
let source = r#"f"{x!s} {x=!r} {x:.3f!r} {{x!r}}""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_nested() {
let source = r#"f"foo {f"bar {x + f"{wow}"}"} baz" f'foo {f'bar'} some {f"another"}'"#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_expression_multiline() {
let source = r#"f"first {
x
*
y
} second""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_multiline() {
let source = r#"f"""
hello
world
""" f'''
world
hello
''' f"some {f"""multiline
allowed {x}"""} string""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_comments() {
let source = r#"f"""
# not a comment { # comment {
x
} # not a comment
""""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_with_ipy_escape_command() {
let source = r#"f"foo {!pwd} bar""#;
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_with_lambda_expression() {
let source = r#"
f"{lambda x:{x}}"
f"{(lambda x:{x})}"
"#
.trim();
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_fstring_with_nul_char() {
let source = r"f'\0'";
assert_debug_snapshot!(lex_source(source));
}
fn lex_fstring_error(source: &str) -> FStringErrorType {
match lex(source, Mode::Module).find_map(std::result::Result::err) {
Some(err) => match err.error {
LexicalErrorType::FStringError(error) => error,
_ => panic!("Expected FStringError: {err:?}"),
},
_ => panic!("Expected atleast one FStringError"),
}
}
#[test]
fn test_fstring_error() {
use FStringErrorType::{
SingleRbrace, UnclosedLbrace, UnterminatedString, UnterminatedTripleQuotedString,
};
assert_eq!(lex_fstring_error("f'}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{{}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{{}}}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'foo}'"), SingleRbrace);
assert_eq!(lex_fstring_error(r"f'\u007b}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{a:b}}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{3:}}>10}'"), SingleRbrace);
assert_eq!(lex_fstring_error(r"f'\{foo}\}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{'"), UnclosedLbrace);
assert_eq!(lex_fstring_error("f'{foo!r'"), UnclosedLbrace);
assert_eq!(lex_fstring_error("f'{foo='"), UnclosedLbrace);
assert_eq!(
lex_fstring_error(
r#"f"{"
"#
),
UnclosedLbrace
);
assert_eq!(lex_fstring_error(r#"f"""{""""#), UnclosedLbrace);
assert_eq!(lex_fstring_error(r#"f""#), UnterminatedString);
assert_eq!(lex_fstring_error(r#"f'"#), UnterminatedString);
assert_eq!(lex_fstring_error(r#"f""""#), UnterminatedTripleQuotedString);
assert_eq!(lex_fstring_error(r#"f'''"#), UnterminatedTripleQuotedString);
assert_eq!(
lex_fstring_error(r#"f"""""#),
UnterminatedTripleQuotedString
);
assert_eq!(
lex_fstring_error(r#"f""""""#),
UnterminatedTripleQuotedString
);
}
}

12
crates/ruff_python_parser/src/lexer/cursor.rs
View file

@ -96,6 +96,18 @@ impl<'a> Cursor<'a> {
}
}
pub(super) fn eat_char3(&mut self, c1: char, c2: char, c3: char) -> bool {
let mut chars = self.chars.clone();
if chars.next() == Some(c1) && chars.next() == Some(c2) && chars.next() == Some(c3) {
self.bump();
self.bump();
self.bump();
true
} else {
false
}
}
pub(super) fn eat_if<F>(&mut self, mut predicate: F) -> Option<char>
where
F: FnMut(char) -> bool,

161
crates/ruff_python_parser/src/lexer/fstring.rs Normal file
View file

@ -0,0 +1,161 @@
use bitflags::bitflags;
use ruff_text_size::TextSize;
bitflags! {
#[derive(Debug)]
pub(crate) struct FStringContextFlags: u8 {
/// The current f-string is a triple-quoted f-string i.e., the number of
/// opening quotes is 3. If this flag is not set, the number of opening
/// quotes is 1.
const TRIPLE = 1 << 0;
/// The current f-string is a double-quoted f-string. If this flag is not
/// set, the current f-string is a single-quoted f-string.
const DOUBLE = 1 << 1;
/// The current f-string is a raw f-string i.e., prefixed with `r`/`R`.
/// If this flag is not set, the current f-string is a normal f-string.
const RAW = 1 << 2;
}
}
/// The context representing the current f-string that the lexer is in.
#[derive(Debug)]
pub(crate) struct FStringContext {
flags: FStringContextFlags,
/// The level of nesting for the lexer when it entered the current f-string.
/// The nesting level includes all kinds of parentheses i.e., round, square,
/// and curly.
nesting: u32,
/// The current depth of format spec for the current f-string. This is because
/// there can be multiple format specs nested for the same f-string.
/// For example, `{a:{b:{c}}}` has 3 format specs.
format_spec_depth: u32,
}
impl FStringContext {
pub(crate) const fn new(flags: FStringContextFlags, nesting: u32) -> Self {
Self {
flags,
nesting,
format_spec_depth: 0,
}
}
pub(crate) const fn nesting(&self) -> u32 {
self.nesting
}
/// Returns the quote character for the current f-string.
pub(crate) const fn quote_char(&self) -> char {
if self.flags.contains(FStringContextFlags::DOUBLE) {
'"'
} else {
'\''
}
}
/// Returns the number of quotes for the current f-string.
pub(crate) const fn quote_size(&self) -> TextSize {
if self.is_triple_quoted() {
TextSize::new(3)
} else {
TextSize::new(1)
}
}
/// Returns the triple quotes for the current f-string if it is a triple-quoted
/// f-string, `None` otherwise.
pub(crate) const fn triple_quotes(&self) -> Option<&'static str> {
if self.is_triple_quoted() {
if self.flags.contains(FStringContextFlags::DOUBLE) {
Some(r#"""""#)
} else {
Some("'''")
}
} else {
None
}
}
/// Returns `true` if the current f-string is a raw f-string.
pub(crate) const fn is_raw_string(&self) -> bool {
self.flags.contains(FStringContextFlags::RAW)
}
/// Returns `true` if the current f-string is a triple-quoted f-string.
pub(crate) const fn is_triple_quoted(&self) -> bool {
self.flags.contains(FStringContextFlags::TRIPLE)
}
/// Calculates the number of open parentheses for the current f-string
/// based on the current level of nesting for the lexer.
const fn open_parentheses_count(&self, current_nesting: u32) -> u32 {
current_nesting.saturating_sub(self.nesting)
}
/// Returns `true` if the lexer is in a f-string expression i.e., between
/// two curly braces.
pub(crate) const fn is_in_expression(&self, current_nesting: u32) -> bool {
self.open_parentheses_count(current_nesting) > self.format_spec_depth
}
/// Returns `true` if the lexer is in a f-string format spec i.e., after a colon.
pub(crate) const fn is_in_format_spec(&self, current_nesting: u32) -> bool {
self.format_spec_depth > 0 && !self.is_in_expression(current_nesting)
}
/// Returns `true` if the context is in a valid position to start format spec
/// i.e., at the same level of nesting as the opening parentheses token.
/// Increments the format spec depth if it is.
///
/// This assumes that the current character for the lexer is a colon (`:`).
pub(crate) fn try_start_format_spec(&mut self, current_nesting: u32) -> bool {
if self
.open_parentheses_count(current_nesting)
.saturating_sub(self.format_spec_depth)
== 1
{
self.format_spec_depth += 1;
true
} else {
false
}
}
/// Decrements the format spec depth if the current f-string is in a format
/// spec.
pub(crate) fn try_end_format_spec(&mut self, current_nesting: u32) {
if self.is_in_format_spec(current_nesting) {
self.format_spec_depth = self.format_spec_depth.saturating_sub(1);
}
}
}
/// The f-strings stack is used to keep track of all the f-strings that the
/// lexer encounters. This is necessary because f-strings can be nested.
#[derive(Debug, Default)]
pub(crate) struct FStrings {
stack: Vec<FStringContext>,
}
impl FStrings {
pub(crate) fn push(&mut self, context: FStringContext) {
self.stack.push(context);
}
pub(crate) fn pop(&mut self) -> Option<FStringContext> {
self.stack.pop()
}
pub(crate) fn current(&self) -> Option<&FStringContext> {
self.stack.last()
}
pub(crate) fn current_mut(&mut self) -> Option<&mut FStringContext> {
self.stack.last_mut()
}
}

5
crates/ruff_python_parser/src/lib.rs
View file

@ -85,7 +85,7 @@
//! return bool(i & 1)
//! "#;
//! let tokens = lex(python_source, Mode::Module);
//! let ast = parse_tokens(tokens, Mode::Module, "<embedded>");
//! let ast = parse_tokens(tokens, python_source, Mode::Module, "<embedded>");
//!
//! assert!(ast.is_ok());
//! ```
@ -146,6 +146,7 @@ pub fn tokenize(contents: &str, mode: Mode) -> Vec<LexResult> {
/// Parse a full Python program from its tokens.
pub fn parse_program_tokens(
lxr: Vec<LexResult>,
source: &str,
source_path: &str,
is_jupyter_notebook: bool,
) -> anyhow::Result<Suite, ParseError> {
@ -154,7 +155,7 @@ pub fn parse_program_tokens(
} else {
Mode::Module
};
match parse_tokens(lxr, mode, source_path)? {
match parse_tokens(lxr, source, mode, source_path)? {
Mod::Module(m) => Ok(m.body),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}

67
crates/ruff_python_parser/src/parser.rs
View file

@ -50,7 +50,7 @@ use ruff_python_ast::{Mod, ModModule, Suite};
/// ```
pub fn parse_program(source: &str, source_path: &str) -> Result<ModModule, ParseError> {
let lexer = lex(source, Mode::Module);
match parse_tokens(lexer, Mode::Module, source_path)? {
match parse_tokens(lexer, source, Mode::Module, source_path)? {
Mod::Module(m) => Ok(m),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}
@ -78,7 +78,7 @@ pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError>
/// ```
pub fn parse_expression(source: &str, source_path: &str) -> Result<ast::Expr, ParseError> {
let lexer = lex(source, Mode::Expression);
match parse_tokens(lexer, Mode::Expression, source_path)? {
match parse_tokens(lexer, source, Mode::Expression, source_path)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
@ -107,7 +107,7 @@ pub fn parse_expression_starts_at(
offset: TextSize,
) -> Result<ast::Expr, ParseError> {
let lexer = lex_starts_at(source, Mode::Module, offset);
match parse_tokens(lexer, Mode::Expression, source_path)? {
match parse_tokens(lexer, source, Mode::Expression, source_path)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
@ -193,7 +193,7 @@ pub fn parse_starts_at(
offset: TextSize,
) -> Result<Mod, ParseError> {
let lxr = lexer::lex_starts_at(source, mode, offset);
parse_tokens(lxr, mode, source_path)
parse_tokens(lxr, source, mode, source_path)
}
/// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
@ -208,11 +208,13 @@ pub fn parse_starts_at(
/// ```
/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
///
/// let expr = parse_tokens(lex("1 + 2", Mode::Expression), Mode::Expression, "<embedded>");
/// let source = "1 + 2";
/// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression, "<embedded>");
/// assert!(expr.is_ok());
/// ```
pub fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
@ -220,6 +222,7 @@ pub fn parse_tokens(
parse_filtered_tokens(
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
source,
mode,
source_path,
)
@ -228,6 +231,7 @@ pub fn parse_tokens(
/// Parse tokens into an AST like [`parse_tokens`], but we already know all tokens are valid.
pub fn parse_ok_tokens(
lxr: impl IntoIterator<Item = Spanned>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
@ -239,12 +243,13 @@ pub fn parse_ok_tokens(
.chain(lxr)
.map(|(t, range)| (range.start(), t, range.end()));
python::TopParser::new()
.parse(mode, lexer)
.parse(source, mode, lexer)
.map_err(|e| parse_error_from_lalrpop(e, source_path))
}
fn parse_filtered_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
@ -252,6 +257,7 @@ fn parse_filtered_tokens(
let lexer = iter::once(Ok(marker_token)).chain(lxr);
python::TopParser::new()
.parse(
source,
mode,
lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
)
@ -1253,11 +1259,58 @@ a = 1
"#
.trim();
let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default());
let parse_err = parse_tokens(lxr, Mode::Module, "<test>").unwrap_err();
let parse_err = parse_tokens(lxr, source, Mode::Module, "<test>").unwrap_err();
assert_eq!(
parse_err.to_string(),
"IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6"
.to_string()
);
}
#[test]
fn test_fstrings() {
let parse_ast = parse_suite(
r#"
f"{" f"}"
f"{foo!s}"
f"{3,}"
f"{3!=4:}"
f'{3:{"}"}>10}'
f'{3:{"{"}>10}'
f"{ foo = }"
f"{ foo = :.3f }"
f"{ foo = !s }"
f"{ 1, 2 = }"
f'{f"{3.1415=:.1f}":*^20}'
{"foo " f"bar {x + y} " "baz": 10}
match foo:
case "foo " f"bar {x + y} " "baz":
pass
f"\{foo}\{bar:\}"
f"\\{{foo\\}}"
"#
.trim(),
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstrings_with_unicode() {
let parse_ast = parse_suite(
r#"
u"foo" f"{bar}" "baz" " some"
"foo" f"{bar}" u"baz" " some"
"foo" f"{bar}" "baz" u" some"
u"foo" f"bar {baz} really" u"bar" "no"
"#
.trim(),
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

127
crates/ruff_python_parser/src/python.lalrpop
View file

@ -3,19 +3,20 @@
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
use ruff_text_size::{Ranged, TextSize};
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
use crate::{
FStringErrorType,
Mode,
lexer::{LexicalError, LexicalErrorType},
function::{ArgumentList, parse_arguments, validate_pos_params, validate_arguments},
context::set_context,
string::parse_strings,
string::{StringType, concatenate_strings, parse_fstring_middle, parse_string_literal},
token::{self, StringKind},
};
use lalrpop_util::ParseError;
grammar(mode: Mode);
grammar(source_code: &str, mode: Mode);
// This is a hack to reduce the amount of lalrpop tables generated:
// For each public entry point, a full parse table is generated.
@ -667,8 +668,8 @@ LiteralPattern: ast::Pattern = {
value: Box::new(value.into()),
range: (location..end_location).into()
}.into(),
<location:@L> <s:(@L string @R)+> <end_location:@R> =>? Ok(ast::PatternMatchValue {
value: Box::new(parse_strings(s)?),
<location:@L> <strings:StringLiteralOrFString+> <end_location:@R> =>? Ok(ast::PatternMatchValue {
value: Box::new(concatenate_strings(strings, (location..end_location).into())?),
range: (location..end_location).into()
}.into()),
}
@ -725,7 +726,7 @@ MappingKey: ast::Expr = {
value: false.into(),
range: (location..end_location).into()
}.into(),
<location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
<location:@L> <strings:StringLiteralOrFString+> <end_location:@R> =>? Ok(concatenate_strings(strings, (location..end_location).into())?),
}
MatchMappingEntry: (ast::Expr, ast::Pattern) = {
@ -1349,7 +1350,13 @@ NamedExpression: ast::ParenthesizedExpr = {
};
LambdaDef: ast::ParenthesizedExpr = {
<location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, StarUntypedParameter>?> <end_location_args:@R> ":" <body:Test<"all">> <end_location:@R> =>? {
<location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, StarUntypedParameter>?> <end_location_args:@R> ":" <fstring_middle:fstring_middle?> <body:Test<"all">> <end_location:@R> =>? {
if fstring_middle.is_some() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location,
})?;
}
parameters.as_ref().map(validate_arguments).transpose()?;
Ok(ast::ExprLambda {
@ -1572,8 +1579,105 @@ SliceOp: Option<ast::ParenthesizedExpr> = {
<location:@L> ":" <e:Test<"all">?> => e,
}
StringLiteralOrFString: StringType = {
StringLiteral,
FStringExpr,
};
StringLiteral: StringType = {
<start_location:@L> <string:string> =>? {
let (source, kind, triple_quoted) = string;
Ok(parse_string_literal(&source, kind, triple_quoted, start_location)?)
}
};
FStringExpr: StringType = {
<location:@L> FStringStart <values:FStringMiddlePattern*> FStringEnd <end_location:@R> => {
StringType::FString(ast::ExprFString {
values,
implicit_concatenated: false,
range: (location..end_location).into()
})
}
};
FStringMiddlePattern: ast::Expr = {
FStringReplacementField,
<start_location:@L> <fstring_middle:fstring_middle> =>? {
let (source, is_raw) = fstring_middle;
Ok(parse_fstring_middle(&source, is_raw, start_location)?)
}
};
FStringReplacementField: ast::Expr = {
<location:@L> "{" <value:TestListOrYieldExpr> <debug:"="?> <conversion:FStringConversion?> <format_spec:FStringFormatSpecSuffix?> "}" <end_location:@R> =>? {
if value.expr.is_lambda_expr() && !value.is_parenthesized() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location: value.start(),
})?;
}
let debug_text = debug.map(|_| {
let start_offset = location + "{".text_len();
let end_offset = if let Some((conversion_start, _)) = conversion {
conversion_start
} else {
format_spec.as_ref().map_or_else(
|| end_location - "}".text_len(),
|spec| spec.range().start() - ":".text_len(),
)
};
ast::DebugText {
leading: source_code[TextRange::new(start_offset, value.range().start())].to_string(),
trailing: source_code[TextRange::new(value.range().end(), end_offset)].to_string(),
}
});
Ok(
ast::ExprFormattedValue {
value: Box::new(value.into()),
debug_text,
conversion: conversion.map_or(ast::ConversionFlag::None, |(_, conversion_flag)| {
conversion_flag
}),
format_spec: format_spec.map(Box::new),
range: (location..end_location).into(),
}
.into()
)
}
};
FStringFormatSpecSuffix: ast::Expr = {
":" <format_spec:FStringFormatSpec> => format_spec
};
FStringFormatSpec: ast::Expr = {
<location:@L> <values:FStringMiddlePattern*> <end_location:@R> => {
ast::ExprFString {
values,
implicit_concatenated: false,
range: (location..end_location).into()
}.into()
},
};
FStringConversion: (TextSize, ast::ConversionFlag) = {
<location:@L> "!" <s:name> =>? {
let conversion = match s.as_str() {
"s" => ast::ConversionFlag::Str,
"r" => ast::ConversionFlag::Repr,
"a" => ast::ConversionFlag::Ascii,
_ => Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
location,
})?
};
Ok((location, conversion))
}
};
Atom<Goal>: ast::ParenthesizedExpr = {
<location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?.into()),
<location:@L> <strings:StringLiteralOrFString+> <end_location:@R> =>? Ok(concatenate_strings(strings, (location..end_location).into())?.into()),
<location:@L> <value:Constant> <end_location:@R> => ast::ExprConstant {
value,
range: (location..end_location).into(),
@ -1842,6 +1946,9 @@ extern {
Dedent => token::Tok::Dedent,
StartModule => token::Tok::StartModule,
StartExpression => token::Tok::StartExpression,
FStringStart => token::Tok::FStringStart,
FStringEnd => token::Tok::FStringEnd,
"!" => token::Tok::Exclamation,
"?" => token::Tok::Question,
"+" => token::Tok::Plus,
"-" => token::Tok::Minus,
@ -1935,6 +2042,10 @@ extern {
kind: <StringKind>,
triple_quoted: <bool>
},
fstring_middle => token::Tok::FStringMiddle {
value: <String>,
is_raw: <bool>
},
name => token::Tok::Name { name: <String> },
ipy_escape_command => token::Tok::IpyEscapeCommand {
kind: <IpyEscapeKind>,

67099
crates/ruff_python_parser/src/python.rs
View file

File diff suppressed because it is too large Load diff

66
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__empty_fstrings.snap Normal file
View file

@ -0,0 +1,66 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringEnd,
2..3,
),
(
String {
value: "",
kind: String,
triple_quoted: false,
},
4..6,
),
(
FStringStart,
7..9,
),
(
FStringEnd,
9..10,
),
(
FStringStart,
11..13,
),
(
FStringEnd,
13..14,
),
(
String {
value: "",
kind: String,
triple_quoted: false,
},
15..17,
),
(
FStringStart,
18..22,
),
(
FStringEnd,
22..25,
),
(
FStringStart,
26..30,
),
(
FStringEnd,
30..33,
),
(
Newline,
33..33,
),
]

88
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring.snap Normal file
View file

@ -0,0 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "normal ",
is_raw: false,
},
2..9,
),
(
Lbrace,
9..10,
),
(
Name {
name: "foo",
},
10..13,
),
(
Rbrace,
13..14,
),
(
FStringMiddle {
value: " {another} ",
is_raw: false,
},
14..27,
),
(
Lbrace,
27..28,
),
(
Name {
name: "bar",
},
28..31,
),
(
Rbrace,
31..32,
),
(
FStringMiddle {
value: " {",
is_raw: false,
},
32..35,
),
(
Lbrace,
35..36,
),
(
Name {
name: "three",
},
36..41,
),
(
Rbrace,
41..42,
),
(
FStringMiddle {
value: "}",
is_raw: false,
},
42..44,
),
(
FStringEnd,
44..45,
),
(
Newline,
45..45,
),
]

60
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap Normal file
View file

@ -0,0 +1,60 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..4,
),
(
FStringMiddle {
value: "\n# not a comment ",
is_raw: false,
},
4..21,
),
(
Lbrace,
21..22,
),
(
Comment(
"# comment {",
),
23..34,
),
(
NonLogicalNewline,
34..35,
),
(
Name {
name: "x",
},
39..40,
),
(
NonLogicalNewline,
40..41,
),
(
Rbrace,
41..42,
),
(
FStringMiddle {
value: " # not a comment\n",
is_raw: false,
},
42..59,
),
(
FStringEnd,
59..62,
),
(
Newline,
62..62,
),
]

116
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_conversion.snap Normal file
View file

@ -0,0 +1,116 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
Lbrace,
2..3,
),
(
Name {
name: "x",
},
3..4,
),
(
Exclamation,
4..5,
),
(
Name {
name: "s",
},
5..6,
),
(
Rbrace,
6..7,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
7..8,
),
(
Lbrace,
8..9,
),
(
Name {
name: "x",
},
9..10,
),
(
Equal,
10..11,
),
(
Exclamation,
11..12,
),
(
Name {
name: "r",
},
12..13,
),
(
Rbrace,
13..14,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
14..15,
),
(
Lbrace,
15..16,
),
(
Name {
name: "x",
},
16..17,
),
(
Colon,
17..18,
),
(
FStringMiddle {
value: ".3f!r",
is_raw: false,
},
18..23,
),
(
Rbrace,
23..24,
),
(
FStringMiddle {
value: " {x!r}",
is_raw: false,
},
24..32,
),
(
FStringEnd,
32..33,
),
(
Newline,
33..33,
),
]

71
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape.snap Normal file
View file

@ -0,0 +1,71 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "\\",
is_raw: false,
},
2..3,
),
(
Lbrace,
3..4,
),
(
Name {
name: "x",
},
4..5,
),
(
Colon,
5..6,
),
(
FStringMiddle {
value: "\\\"\\",
is_raw: false,
},
6..9,
),
(
Lbrace,
9..10,
),
(
Name {
name: "x",
},
10..11,
),
(
Rbrace,
11..12,
),
(
Rbrace,
12..13,
),
(
FStringMiddle {
value: " \\\"\\\"\\\n end",
is_raw: false,
},
13..24,
),
(
FStringEnd,
24..25,
),
(
Newline,
25..25,
),
]

98
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape_braces.snap Normal file
View file

@ -0,0 +1,98 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "\\",
is_raw: false,
},
2..3,
),
(
Lbrace,
3..4,
),
(
Name {
name: "foo",
},
4..7,
),
(
Rbrace,
7..8,
),
(
FStringEnd,
8..9,
),
(
FStringStart,
10..12,
),
(
FStringMiddle {
value: "\\\\",
is_raw: false,
},
12..14,
),
(
Lbrace,
14..15,
),
(
Name {
name: "foo",
},
15..18,
),
(
Rbrace,
18..19,
),
(
FStringEnd,
19..20,
),
(
FStringStart,
21..23,
),
(
FStringMiddle {
value: "\\{foo}",
is_raw: false,
},
23..31,
),
(
FStringEnd,
31..32,
),
(
FStringStart,
33..35,
),
(
FStringMiddle {
value: "\\\\{foo}",
is_raw: false,
},
35..44,
),
(
FStringEnd,
44..45,
),
(
Newline,
45..45,
),
]

71
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape_raw.snap Normal file
View file

@ -0,0 +1,71 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..3,
),
(
FStringMiddle {
value: "\\",
is_raw: true,
},
3..4,
),
(
Lbrace,
4..5,
),
(
Name {
name: "x",
},
5..6,
),
(
Colon,
6..7,
),
(
FStringMiddle {
value: "\\\"\\",
is_raw: true,
},
7..10,
),
(
Lbrace,
10..11,
),
(
Name {
name: "x",
},
11..12,
),
(
Rbrace,
12..13,
),
(
Rbrace,
13..14,
),
(
FStringMiddle {
value: " \\\"\\\"\\\n end",
is_raw: true,
},
14..25,
),
(
FStringEnd,
25..26,
),
(
Newline,
26..26,
),
]

72
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_expression_multiline.snap Normal file
View file

@ -0,0 +1,72 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "first ",
is_raw: false,
},
2..8,
),
(
Lbrace,
8..9,
),
(
NonLogicalNewline,
9..10,
),
(
Name {
name: "x",
},
14..15,
),
(
NonLogicalNewline,
15..16,
),
(
Star,
24..25,
),
(
NonLogicalNewline,
25..26,
),
(
Name {
name: "y",
},
38..39,
),
(
NonLogicalNewline,
39..40,
),
(
Rbrace,
40..41,
),
(
FStringMiddle {
value: " second",
is_raw: false,
},
41..48,
),
(
FStringEnd,
48..49,
),
(
Newline,
49..49,
),
]

99
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_multiline.snap Normal file
View file

@ -0,0 +1,99 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..4,
),
(
FStringMiddle {
value: "\nhello\n world\n",
is_raw: false,
},
4..21,
),
(
FStringEnd,
21..24,
),
(
FStringStart,
25..29,
),
(
FStringMiddle {
value: "\n world\nhello\n",
is_raw: false,
},
29..46,
),
(
FStringEnd,
46..49,
),
(
FStringStart,
50..52,
),
(
FStringMiddle {
value: "some ",
is_raw: false,
},
52..57,
),
(
Lbrace,
57..58,
),
(
FStringStart,
58..62,
),
(
FStringMiddle {
value: "multiline\nallowed ",
is_raw: false,
},
62..80,
),
(
Lbrace,
80..81,
),
(
Name {
name: "x",
},
81..82,
),
(
Rbrace,
82..83,
),
(
FStringEnd,
83..86,
),
(
Rbrace,
86..87,
),
(
FStringMiddle {
value: " string",
is_raw: false,
},
87..94,
),
(
FStringEnd,
94..95,
),
(
Newline,
95..95,
),
]

25
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_named_unicode.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "\\N{BULLET} normal \\Nope \\N",
is_raw: false,
},
2..28,
),
(
FStringEnd,
28..29,
),
(
Newline,
29..29,
),
]

46
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_named_unicode_raw.snap Normal file
View file

@ -0,0 +1,46 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..3,
),
(
FStringMiddle {
value: "\\N",
is_raw: true,
},
3..5,
),
(
Lbrace,
5..6,
),
(
Name {
name: "BULLET",
},
6..12,
),
(
Rbrace,
12..13,
),
(
FStringMiddle {
value: " normal",
is_raw: true,
},
13..20,
),
(
FStringEnd,
20..21,
),
(
Newline,
21..21,
),
]

163
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_nested.snap Normal file
View file

@ -0,0 +1,163 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "foo ",
is_raw: false,
},
2..6,
),
(
Lbrace,
6..7,
),
(
FStringStart,
7..9,
),
(
FStringMiddle {
value: "bar ",
is_raw: false,
},
9..13,
),
(
Lbrace,
13..14,
),
(
Name {
name: "x",
},
14..15,
),
(
Plus,
16..17,
),
(
FStringStart,
18..20,
),
(
Lbrace,
20..21,
),
(
Name {
name: "wow",
},
21..24,
),
(
Rbrace,
24..25,
),
(
FStringEnd,
25..26,
),
(
Rbrace,
26..27,
),
(
FStringEnd,
27..28,
),
(
Rbrace,
28..29,
),
(
FStringMiddle {
value: " baz",
is_raw: false,
},
29..33,
),
(
FStringEnd,
33..34,
),
(
FStringStart,
35..37,
),
(
FStringMiddle {
value: "foo ",
is_raw: false,
},
37..41,
),
(
Lbrace,
41..42,
),
(
FStringStart,
42..44,
),
(
FStringMiddle {
value: "bar",
is_raw: false,
},
44..47,
),
(
FStringEnd,
47..48,
),
(
Rbrace,
48..49,
),
(
FStringMiddle {
value: " some ",
is_raw: false,
},
49..55,
),
(
Lbrace,
55..56,
),
(
FStringStart,
56..58,
),
(
FStringMiddle {
value: "another",
is_raw: false,
},
58..65,
),
(
FStringEnd,
65..66,
),
(
Rbrace,
66..67,
),
(
FStringEnd,
67..68,
),
(
Newline,
68..68,
),
]

154
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_parentheses.snap Normal file
View file

@ -0,0 +1,154 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
Lbrace,
2..3,
),
(
Rbrace,
3..4,
),
(
FStringEnd,
4..5,
),
(
FStringStart,
6..8,
),
(
FStringMiddle {
value: "{}",
is_raw: false,
},
8..12,
),
(
FStringEnd,
12..13,
),
(
FStringStart,
14..16,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
16..17,
),
(
Lbrace,
17..18,
),
(
Rbrace,
18..19,
),
(
FStringEnd,
19..20,
),
(
FStringStart,
21..23,
),
(
FStringMiddle {
value: "{",
is_raw: false,
},
23..25,
),
(
Lbrace,
25..26,
),
(
Rbrace,
26..27,
),
(
FStringMiddle {
value: "}",
is_raw: false,
},
27..29,
),
(
FStringEnd,
29..30,
),
(
FStringStart,
31..33,
),
(
FStringMiddle {
value: "{{}}",
is_raw: false,
},
33..41,
),
(
FStringEnd,
41..42,
),
(
FStringStart,
43..45,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
45..46,
),
(
Lbrace,
46..47,
),
(
Rbrace,
47..48,
),
(
FStringMiddle {
value: " {} {",
is_raw: false,
},
48..56,
),
(
Lbrace,
56..57,
),
(
Rbrace,
57..58,
),
(
FStringMiddle {
value: "} {{}} ",
is_raw: false,
},
58..71,
),
(
FStringEnd,
71..72,
),
(
Newline,
72..72,
),
]

90
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_prefix.snap Normal file
View file

@ -0,0 +1,90 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringEnd,
2..3,
),
(
FStringStart,
4..6,
),
(
FStringEnd,
6..7,
),
(
FStringStart,
8..11,
),
(
FStringEnd,
11..12,
),
(
FStringStart,
13..16,
),
(
FStringEnd,
16..17,
),
(
FStringStart,
18..21,
),
(
FStringEnd,
21..22,
),
(
FStringStart,
23..26,
),
(
FStringEnd,
26..27,
),
(
FStringStart,
28..31,
),
(
FStringEnd,
31..32,
),
(
FStringStart,
33..36,
),
(
FStringEnd,
36..37,
),
(
FStringStart,
38..41,
),
(
FStringEnd,
41..42,
),
(
FStringStart,
43..46,
),
(
FStringEnd,
46..47,
),
(
Newline,
47..47,
),
]

201
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_format_spec.snap Normal file
View file

@ -0,0 +1,201 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
Lbrace,
2..3,
),
(
Name {
name: "foo",
},
3..6,
),
(
Colon,
6..7,
),
(
Rbrace,
7..8,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
8..9,
),
(
Lbrace,
9..10,
),
(
Name {
name: "x",
},
10..11,
),
(
Equal,
11..12,
),
(
Exclamation,
12..13,
),
(
Name {
name: "s",
},
13..14,
),
(
Colon,
14..15,
),
(
FStringMiddle {
value: ".3f",
is_raw: false,
},
15..18,
),
(
Rbrace,
18..19,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
19..20,
),
(
Lbrace,
20..21,
),
(
Name {
name: "x",
},
21..22,
),
(
Colon,
22..23,
),
(
FStringMiddle {
value: ".",
is_raw: false,
},
23..24,
),
(
Lbrace,
24..25,
),
(
Name {
name: "y",
},
25..26,
),
(
Rbrace,
26..27,
),
(
FStringMiddle {
value: "f",
is_raw: false,
},
27..28,
),
(
Rbrace,
28..29,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
29..30,
),
(
Lbrace,
30..31,
),
(
String {
value: "",
kind: String,
triple_quoted: false,
},
31..33,
),
(
Colon,
33..34,
),
(
FStringMiddle {
value: "*^",
is_raw: false,
},
34..36,
),
(
Lbrace,
36..37,
),
(
Int {
value: 1,
},
37..38,
),
(
Colon,
38..39,
),
(
Lbrace,
39..40,
),
(
Int {
value: 1,
},
40..41,
),
(
Rbrace,
41..42,
),
(
Rbrace,
42..43,
),
(
Rbrace,
43..44,
),
(
FStringEnd,
44..45,
),
(
Newline,
45..45,
),
]

50
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_ipy_escape_command.snap Normal file
View file

@ -0,0 +1,50 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "foo ",
is_raw: false,
},
2..6,
),
(
Lbrace,
6..7,
),
(
Exclamation,
7..8,
),
(
Name {
name: "pwd",
},
8..11,
),
(
Rbrace,
11..12,
),
(
FStringMiddle {
value: " bar",
is_raw: false,
},
12..16,
),
(
FStringEnd,
16..17,
),
(
Newline,
17..17,
),
]

110
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_lambda_expression.snap Normal file
View file

@ -0,0 +1,110 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
Lbrace,
2..3,
),
(
Lambda,
3..9,
),
(
Name {
name: "x",
},
10..11,
),
(
Colon,
11..12,
),
(
Lbrace,
12..13,
),
(
Name {
name: "x",
},
13..14,
),
(
Rbrace,
14..15,
),
(
Rbrace,
15..16,
),
(
FStringEnd,
16..17,
),
(
Newline,
17..18,
),
(
FStringStart,
18..20,
),
(
Lbrace,
20..21,
),
(
Lpar,
21..22,
),
(
Lambda,
22..28,
),
(
Name {
name: "x",
},
29..30,
),
(
Colon,
30..31,
),
(
Lbrace,
31..32,
),
(
Name {
name: "x",
},
32..33,
),
(
Rbrace,
33..34,
),
(
Rpar,
34..35,
),
(
Rbrace,
35..36,
),
(
FStringEnd,
36..37,
),
(
Newline,
37..37,
),
]

170
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_named_expression.snap Normal file
View file

@ -0,0 +1,170 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
Lbrace,
2..3,
),
(
Name {
name: "x",
},
3..4,
),
(
Colon,
4..5,
),
(
FStringMiddle {
value: "=10",
is_raw: false,
},
5..8,
),
(
Rbrace,
8..9,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
9..10,
),
(
Lbrace,
10..11,
),
(
Lpar,
11..12,
),
(
Name {
name: "x",
},
12..13,
),
(
ColonEqual,
13..15,
),
(
Int {
value: 10,
},
15..17,
),
(
Rpar,
17..18,
),
(
Rbrace,
18..19,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
19..20,
),
(
Lbrace,
20..21,
),
(
Name {
name: "x",
},
21..22,
),
(
Comma,
22..23,
),
(
Lbrace,
23..24,
),
(
Name {
name: "y",
},
24..25,
),
(
ColonEqual,
25..27,
),
(
Int {
value: 10,
},
27..29,
),
(
Rbrace,
29..30,
),
(
Rbrace,
30..31,
),
(
FStringMiddle {
value: " ",
is_raw: false,
},
31..32,
),
(
Lbrace,
32..33,
),
(
Lsqb,
33..34,
),
(
Name {
name: "x",
},
34..35,
),
(
ColonEqual,
35..37,
),
(
Int {
value: 10,
},
37..39,
),
(
Rsqb,
39..40,
),
(
Rbrace,
40..41,
),
(
FStringEnd,
41..42,
),
(
Newline,
42..42,
),
]

25
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_nul_char.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "\\0",
is_raw: false,
},
2..4,
),
(
FStringEnd,
4..5,
),
(
Newline,
5..5,
),
]

848
crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap Normal file
View file

@ -0,0 +1,848 @@
---
source: crates/ruff_python_parser/src/parser.rs
expression: parse_ast
---
[
Expr(
StmtExpr {
range: 0..9,
value: FString(
ExprFString {
range: 0..9,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..8,
value: Constant(
ExprConstant {
range: 3..7,
value: Str(
StringConstant {
value: " f",
unicode: false,
implicit_concatenated: false,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 10..20,
value: FString(
ExprFString {
range: 10..20,
values: [
FormattedValue(
ExprFormattedValue {
range: 12..19,
value: Name(
ExprName {
range: 13..16,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: Str,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 21..28,
value: FString(
ExprFString {
range: 21..28,
values: [
FormattedValue(
ExprFormattedValue {
range: 23..27,
value: Tuple(
ExprTuple {
range: 24..26,
elts: [
Constant(
ExprConstant {
range: 24..25,
value: Int(
3,
),
},
),
],
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 29..39,
value: FString(
ExprFString {
range: 29..39,
values: [
FormattedValue(
ExprFormattedValue {
range: 31..38,
value: Compare(
ExprCompare {
range: 32..36,
left: Constant(
ExprConstant {
range: 32..33,
value: Int(
3,
),
},
),
ops: [
NotEq,
],
comparators: [
Constant(
ExprConstant {
range: 35..36,
value: Int(
4,
),
},
),
],
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 37..37,
values: [],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 40..55,
value: FString(
ExprFString {
range: 40..55,
values: [
FormattedValue(
ExprFormattedValue {
range: 42..54,
value: Constant(
ExprConstant {
range: 43..44,
value: Int(
3,
),
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 45..53,
values: [
FormattedValue(
ExprFormattedValue {
range: 45..50,
value: Constant(
ExprConstant {
range: 46..49,
value: Str(
StringConstant {
value: "}",
unicode: false,
implicit_concatenated: false,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 50..53,
value: Str(
StringConstant {
value: ">10",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 56..71,
value: FString(
ExprFString {
range: 56..71,
values: [
FormattedValue(
ExprFormattedValue {
range: 58..70,
value: Constant(
ExprConstant {
range: 59..60,
value: Int(
3,
),
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 61..69,
values: [
FormattedValue(
ExprFormattedValue {
range: 61..66,
value: Constant(
ExprConstant {
range: 62..65,
value: Str(
StringConstant {
value: "{",
unicode: false,
implicit_concatenated: false,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 66..69,
value: Str(
StringConstant {
value: ">10",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 72..86,
value: FString(
ExprFString {
range: 72..86,
values: [
FormattedValue(
ExprFormattedValue {
range: 74..85,
value: Name(
ExprName {
range: 77..80,
id: "foo",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: " ",
trailing: " = ",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 87..107,
value: FString(
ExprFString {
range: 87..107,
values: [
FormattedValue(
ExprFormattedValue {
range: 89..106,
value: Name(
ExprName {
range: 92..95,
id: "foo",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: " ",
trailing: " = ",
},
),
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 100..105,
values: [
Constant(
ExprConstant {
range: 100..105,
value: Str(
StringConstant {
value: ".3f ",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 108..126,
value: FString(
ExprFString {
range: 108..126,
values: [
FormattedValue(
ExprFormattedValue {
range: 110..125,
value: Name(
ExprName {
range: 113..116,
id: "foo",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: " ",
trailing: " = ",
},
),
conversion: Str,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 127..143,
value: FString(
ExprFString {
range: 127..143,
values: [
FormattedValue(
ExprFormattedValue {
range: 129..142,
value: Tuple(
ExprTuple {
range: 132..136,
elts: [
Constant(
ExprConstant {
range: 132..133,
value: Int(
1,
),
},
),
Constant(
ExprConstant {
range: 135..136,
value: Int(
2,
),
},
),
],
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: " ",
trailing: " = ",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 144..170,
value: FString(
ExprFString {
range: 144..170,
values: [
FormattedValue(
ExprFormattedValue {
range: 146..169,
value: FString(
ExprFString {
range: 147..163,
values: [
FormattedValue(
ExprFormattedValue {
range: 149..162,
value: Constant(
ExprConstant {
range: 150..156,
value: Float(
3.1415,
),
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 158..161,
values: [
Constant(
ExprConstant {
range: 158..161,
value: Str(
StringConstant {
value: ".1f",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 164..168,
values: [
Constant(
ExprConstant {
range: 164..168,
value: Str(
StringConstant {
value: "*^20",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 172..206,
value: Dict(
ExprDict {
range: 172..206,
keys: [
Some(
FString(
ExprFString {
range: 173..201,
values: [
Constant(
ExprConstant {
range: 174..186,
value: Str(
StringConstant {
value: "foo bar ",
unicode: false,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 186..193,
value: BinOp(
ExprBinOp {
range: 187..192,
left: Name(
ExprName {
range: 187..188,
id: "x",
ctx: Load,
},
),
op: Add,
right: Name(
ExprName {
range: 191..192,
id: "y",
ctx: Load,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 193..200,
value: Str(
StringConstant {
value: " baz",
unicode: false,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
),
],
values: [
Constant(
ExprConstant {
range: 203..205,
value: Int(
10,
),
},
),
],
},
),
},
),
Match(
StmtMatch {
range: 207..269,
subject: Name(
ExprName {
range: 213..216,
id: "foo",
ctx: Load,
},
),
cases: [
MatchCase {
range: 222..269,
pattern: MatchValue(
PatternMatchValue {
range: 227..255,
value: FString(
ExprFString {
range: 227..255,
values: [
Constant(
ExprConstant {
range: 228..240,
value: Str(
StringConstant {
value: "foo bar ",
unicode: false,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 240..247,
value: BinOp(
ExprBinOp {
range: 241..246,
left: Name(
ExprName {
range: 241..242,
id: "x",
ctx: Load,
},
),
op: Add,
right: Name(
ExprName {
range: 245..246,
id: "y",
ctx: Load,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 247..254,
value: Str(
StringConstant {
value: " baz",
unicode: false,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
},
),
guard: None,
body: [
Pass(
StmtPass {
range: 265..269,
},
),
],
},
],
},
),
Expr(
StmtExpr {
range: 271..288,
value: FString(
ExprFString {
range: 271..288,
values: [
Constant(
ExprConstant {
range: 273..274,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 274..279,
value: Name(
ExprName {
range: 275..278,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 279..280,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 280..287,
value: Name(
ExprName {
range: 281..284,
id: "bar",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 285..286,
values: [
Constant(
ExprConstant {
range: 285..286,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 289..303,
value: FString(
ExprFString {
range: 289..303,
values: [
Constant(
ExprConstant {
range: 291..302,
value: Str(
StringConstant {
value: "\\{foo\\}",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
},
),
]

214
crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings_with_unicode.snap Normal file
View file

@ -0,0 +1,214 @@
---
source: crates/ruff_python_parser/src/parser.rs
expression: parse_ast
---
[
Expr(
StmtExpr {
range: 0..29,
value: FString(
ExprFString {
range: 0..29,
values: [
Constant(
ExprConstant {
range: 2..5,
value: Str(
StringConstant {
value: "foo",
unicode: true,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 9..14,
value: Name(
ExprName {
range: 10..13,
id: "bar",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 17..28,
value: Str(
StringConstant {
value: "baz some",
unicode: false,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
},
),
Expr(
StmtExpr {
range: 30..59,
value: FString(
ExprFString {
range: 30..59,
values: [
Constant(
ExprConstant {
range: 31..34,
value: Str(
StringConstant {
value: "foo",
unicode: false,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 38..43,
value: Name(
ExprName {
range: 39..42,
id: "bar",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 47..58,
value: Str(
StringConstant {
value: "baz some",
unicode: true,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
},
),
Expr(
StmtExpr {
range: 60..89,
value: FString(
ExprFString {
range: 60..89,
values: [
Constant(
ExprConstant {
range: 61..64,
value: Str(
StringConstant {
value: "foo",
unicode: false,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 68..73,
value: Name(
ExprName {
range: 69..72,
id: "bar",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 76..88,
value: Str(
StringConstant {
value: "baz some",
unicode: false,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
},
),
Expr(
StmtExpr {
range: 90..128,
value: FString(
ExprFString {
range: 90..128,
values: [
Constant(
ExprConstant {
range: 92..103,
value: Str(
StringConstant {
value: "foobar ",
unicode: true,
implicit_concatenated: true,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 103..108,
value: Name(
ExprName {
range: 104..107,
id: "baz",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 108..127,
value: Str(
StringConstant {
value: " reallybarno",
unicode: false,
implicit_concatenated: true,
},
),
},
),
],
implicit_concatenated: true,
},
),
},
),
]

45
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_base.snap
View file

@ -3,24 +3,37 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..7,
id: "user",
ctx: Load,
Expr(
StmtExpr {
range: 0..10,
value: FString(
ExprFString {
range: 0..10,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..7,
id: "user",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
]

131
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_base_more.snap
View file

@ -3,68 +3,81 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
Constant(
ExprConstant {
range: 2..6,
value: Str(
StringConstant {
value: "mix ",
unicode: false,
Expr(
StmtExpr {
range: 0..38,
value: FString(
ExprFString {
range: 0..38,
values: [
Constant(
ExprConstant {
range: 2..6,
value: Str(
StringConstant {
value: "mix ",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 6..13,
value: Name(
ExprName {
range: 7..11,
id: "user",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 13..28,
value: Str(
StringConstant {
value: " with text and ",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 28..37,
value: Name(
ExprName {
range: 29..35,
id: "second",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 6..13,
value: Name(
ExprName {
range: 7..11,
id: "user",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 13..28,
value: Str(
StringConstant {
value: " with text and ",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 28..37,
value: Name(
ExprName {
range: 29..35,
id: "second",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: None,
},
),
]

83
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_format.snap
View file

@ -3,44 +3,57 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..13,
value: Name(
ExprName {
range: 3..7,
id: "user",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 9..12,
values: [
Constant(
ExprConstant {
range: 9..12,
value: Str(
StringConstant {
value: ">10",
unicode: false,
Expr(
StmtExpr {
range: 0..14,
value: FString(
ExprFString {
range: 0..14,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..13,
value: Name(
ExprName {
range: 3..7,
id: "user",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "=",
},
),
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 9..12,
values: [
Constant(
ExprConstant {
range: 9..12,
value: Str(
StringConstant {
value: ">10",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),

17
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_empty_fstring.snap
View file

@ -1,5 +1,18 @@
---
source: crates/ruff_python_parser/src/string.rs
expression: "parse_fstring(\"\").unwrap()"
expression: "parse_suite(r#\"f\"\"\"#, \"<test>\").unwrap()"
---
[]
[
Expr(
StmtExpr {
range: 0..3,
value: FString(
ExprFString {
range: 0..3,
values: [],
implicit_concatenated: false,
},
),
},
),
]

87
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring.snap
View file

@ -3,43 +3,56 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..5,
value: Name(
ExprName {
range: 3..4,
id: "a",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
FormattedValue(
ExprFormattedValue {
range: 5..10,
value: Name(
ExprName {
range: 7..8,
id: "b",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 10..17,
value: Str(
StringConstant {
value: "{foo}",
unicode: false,
Expr(
StmtExpr {
range: 0..18,
value: FString(
ExprFString {
range: 0..18,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..5,
value: Name(
ExprName {
range: 3..4,
id: "a",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
FormattedValue(
ExprFormattedValue {
range: 5..10,
value: Name(
ExprName {
range: 7..8,
id: "b",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 10..17,
value: Str(
StringConstant {
value: "{foo}",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),

65
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap
View file

@ -3,38 +3,51 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Compare(
ExprCompare {
range: 3..11,
left: Constant(
ExprConstant {
range: 3..5,
value: Int(
42,
),
},
),
ops: [
Eq,
],
comparators: [
Constant(
ExprConstant {
range: 9..11,
value: Int(
42,
Expr(
StmtExpr {
range: 0..13,
value: FString(
ExprFString {
range: 0..13,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Compare(
ExprCompare {
range: 3..11,
left: Constant(
ExprConstant {
range: 3..5,
value: Int(
42,
),
},
),
ops: [
Eq,
],
comparators: [
Constant(
ExprConstant {
range: 9..11,
value: Int(
42,
),
},
),
],
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
]

91
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_concatenation_string_spec.snap
View file

@ -3,47 +3,60 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..15,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..14,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..14,
value: Constant(
ExprConstant {
range: 8..13,
value: Str(
StringConstant {
value: "",
unicode: false,
implicit_concatenated: true,
},
),
Expr(
StmtExpr {
range: 0..16,
value: FString(
ExprFString {
range: 0..16,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..15,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..14,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..14,
value: Constant(
ExprConstant {
range: 8..13,
value: Str(
StringConstant {
value: "",
unicode: false,
implicit_concatenated: true,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),

81
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_spec.snap
View file

@ -3,42 +3,55 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..14,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..13,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..13,
value: Name(
ExprName {
range: 8..12,
id: "spec",
ctx: Load,
Expr(
StmtExpr {
range: 0..15,
value: FString(
ExprFString {
range: 0..15,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..14,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..13,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..13,
value: Name(
ExprName {
range: 8..12,
id: "spec",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),

91
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_string_spec.snap
View file

@ -3,47 +3,60 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..11,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..11,
value: Constant(
ExprConstant {
range: 8..10,
value: Str(
StringConstant {
value: "",
unicode: false,
implicit_concatenated: false,
},
),
Expr(
StmtExpr {
range: 0..13,
value: FString(
ExprFString {
range: 0..13,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..11,
values: [
FormattedValue(
ExprFormattedValue {
range: 7..11,
value: Constant(
ExprConstant {
range: 8..10,
value: Str(
StringConstant {
value: "",
unicode: false,
implicit_concatenated: false,
},
),
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),

65
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap
View file

@ -3,38 +3,51 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..10,
value: Compare(
ExprCompare {
range: 3..9,
left: Constant(
ExprConstant {
range: 3..4,
value: Int(
1,
),
},
),
ops: [
NotEq,
],
comparators: [
Constant(
ExprConstant {
range: 8..9,
value: Int(
2,
Expr(
StmtExpr {
range: 0..11,
value: FString(
ExprFString {
range: 0..11,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..10,
value: Compare(
ExprCompare {
range: 3..9,
left: Constant(
ExprConstant {
range: 3..4,
value: Int(
1,
),
},
),
ops: [
NotEq,
],
comparators: [
Constant(
ExprConstant {
range: 8..9,
value: Int(
2,
),
},
),
],
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
]

73
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_nested_spec.snap
View file

@ -3,39 +3,52 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..11,
values: [
Constant(
ExprConstant {
range: 7..11,
value: Str(
StringConstant {
value: "spec",
unicode: false,
Expr(
StmtExpr {
range: 0..13,
value: FString(
ExprFString {
range: 0..13,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..12,
value: Name(
ExprName {
range: 3..6,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 7..11,
values: [
Constant(
ExprConstant {
range: 7..11,
value: Str(
StringConstant {
value: "spec",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),

45
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_self_doc_prec_space.snap
View file

@ -3,24 +3,37 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..4,
id: "x",
ctx: Load,
Expr(
StmtExpr {
range: 0..10,
value: FString(
ExprFString {
range: 0..10,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..4,
id: "x",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: " =",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: " =",
},
),
conversion: None,
format_spec: None,
},
),
]

45
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_self_doc_trailing_space.snap
View file

@ -3,24 +3,37 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..4,
id: "x",
ctx: Load,
Expr(
StmtExpr {
range: 0..10,
value: FString(
ExprFString {
range: 0..10,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Name(
ExprName {
range: 3..4,
id: "x",
ctx: Load,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "= ",
},
),
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: Some(
DebugText {
leading: "",
trailing: "= ",
},
),
conversion: None,
format_spec: None,
},
),
]

33
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_yield_expr.snap
View file

@ -3,18 +3,31 @@ source: crates/ruff_python_parser/src/string.rs
expression: parse_ast
---
[
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Yield(
ExprYield {
range: 3..8,
value: None,
Expr(
StmtExpr {
range: 0..10,
value: FString(
ExprFString {
range: 0..10,
values: [
FormattedValue(
ExprFormattedValue {
range: 2..9,
value: Yield(
ExprYield {
range: 3..8,
value: None,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
implicit_concatenated: false,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
]

796
crates/ruff_python_parser/src/string.rs
View file

File diff suppressed because it is too large Load diff

85
crates/ruff_python_parser/src/token.rs
View file

@ -44,6 +44,19 @@ pub enum Tok {
/// Whether the string is triple quoted.
triple_quoted: bool,
},
/// Token value for the start of an f-string. This includes the `f`/`F`/`fr` prefix
/// and the opening quote(s).
FStringStart,
/// Token value that includes the portion of text inside the f-string that's not
/// part of the expression part and isn't an opening or closing brace.
FStringMiddle {
/// The string value.
value: String,
/// Whether the string is raw or not.
is_raw: bool,
},
/// Token value for the end of an f-string. This includes the closing quote.
FStringEnd,
/// Token value for IPython escape commands. These are recognized by the lexer
/// only when the mode is [`Mode::Ipython`].
IpyEscapeCommand {
@ -66,6 +79,8 @@ pub enum Tok {
EndOfFile,
/// Token value for a question mark `?`. This is only used in [`Mode::Ipython`].
Question,
/// Token value for a exclamation mark `!`.
Exclamation,
/// Token value for a left parenthesis `(`.
Lpar,
/// Token value for a right parenthesis `)`.
@ -234,6 +249,9 @@ impl fmt::Display for Tok {
let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
write!(f, "{kind}{quotes}{value}{quotes}")
}
FStringStart => f.write_str("FStringStart"),
FStringMiddle { value, .. } => f.write_str(value),
FStringEnd => f.write_str("FStringEnd"),
IpyEscapeCommand { kind, value } => write!(f, "{kind}{value}"),
Newline => f.write_str("Newline"),
NonLogicalNewline => f.write_str("NonLogicalNewline"),
@ -243,6 +261,7 @@ impl fmt::Display for Tok {
StartExpression => f.write_str("StartExpression"),
EndOfFile => f.write_str("EOF"),
Question => f.write_str("'?'"),
Exclamation => f.write_str("'!'"),
Lpar => f.write_str("'('"),
Rpar => f.write_str("')'"),
Lsqb => f.write_str("'['"),
@ -336,19 +355,19 @@ impl fmt::Display for Tok {
/// The kind of string literal as described in the [String and Bytes literals]
/// section of the Python reference.
///
/// Note that f-strings are not included here, because as of [PEP 701] they
/// emit different tokens than other string literals.
///
/// [String and Bytes literals]: https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
/// [PEP 701]: https://peps.python.org/pep-0701/
#[derive(PartialEq, Eq, Debug, Clone, Hash, Copy)] // TODO: is_macro::Is
pub enum StringKind {
/// A normal string literal with no prefix.
String,
/// A f-string literal, with a `f` or `F` prefix.
FString,
/// A byte string literal, with a `b` or `B` prefix.
Bytes,
/// A raw string literal, with a `r` or `R` prefix.
RawString,
/// A raw f-string literal, with a `rf`/`fr` or `rF`/`Fr` or `Rf`/`fR` or `RF`/`FR` prefix.
RawFString,
/// A raw byte string literal, with a `rb`/`br` or `rB`/`Br` or `Rb`/`bR` or `RB`/`BR` prefix.
RawBytes,
/// A unicode string literal, with a `u` or `U` prefix.
@ -361,7 +380,6 @@ impl TryFrom<char> for StringKind {
fn try_from(ch: char) -> Result<Self, String> {
match ch {
'r' | 'R' => Ok(StringKind::RawString),
'f' | 'F' => Ok(StringKind::FString),
'u' | 'U' => Ok(StringKind::Unicode),
'b' | 'B' => Ok(StringKind::Bytes),
c => Err(format!("Unexpected string prefix: {c}")),
@ -374,8 +392,6 @@ impl TryFrom<[char; 2]> for StringKind {
fn try_from(chars: [char; 2]) -> Result<Self, String> {
match chars {
['r' | 'R', 'f' | 'F'] => Ok(StringKind::RawFString),
['f' | 'F', 'r' | 'R'] => Ok(StringKind::RawFString),
['r' | 'R', 'b' | 'B'] => Ok(StringKind::RawBytes),
['b' | 'B', 'r' | 'R'] => Ok(StringKind::RawBytes),
[c1, c2] => Err(format!("Unexpected string prefix: {c1}{c2}")),
@ -385,32 +401,16 @@ impl TryFrom<[char; 2]> for StringKind {
impl fmt::Display for StringKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use StringKind::{Bytes, FString, RawBytes, RawFString, RawString, String, Unicode};
match self {
String => f.write_str(""),
FString => f.write_str("f"),
Bytes => f.write_str("b"),
RawString => f.write_str("r"),
RawFString => f.write_str("rf"),
RawBytes => f.write_str("rb"),
Unicode => f.write_str("u"),
}
f.write_str(self.as_str())
}
}
impl StringKind {
/// Returns true if the string is a raw string, i,e one of
/// [`StringKind::RawString`] or [`StringKind::RawFString`] or [`StringKind::RawBytes`].
/// [`StringKind::RawString`] or [`StringKind::RawBytes`].
pub fn is_raw(&self) -> bool {
use StringKind::{RawBytes, RawFString, RawString};
matches!(self, RawString | RawFString | RawBytes)
}
/// Returns true if the string is an f-string, i,e one of
/// [`StringKind::FString`] or [`StringKind::RawFString`].
pub fn is_any_fstring(&self) -> bool {
use StringKind::{FString, RawFString};
matches!(self, FString | RawFString)
use StringKind::{RawBytes, RawString};
matches!(self, RawString | RawBytes)
}
/// Returns true if the string is a byte string, i,e one of
@ -427,14 +427,25 @@ impl StringKind {
/// Returns the number of characters in the prefix.
pub fn prefix_len(&self) -> TextSize {
use StringKind::{Bytes, FString, RawBytes, RawFString, RawString, String, Unicode};
use StringKind::{Bytes, RawBytes, RawString, String, Unicode};
let len = match self {
String => 0,
RawString | FString | Unicode | Bytes => 1,
RawFString | RawBytes => 2,
RawString | Unicode | Bytes => 1,
RawBytes => 2,
};
len.into()
}
pub fn as_str(&self) -> &'static str {
use StringKind::{Bytes, RawBytes, RawString, String, Unicode};
match self {
String => "",
Bytes => "b",
RawString => "r",
RawBytes => "rb",
Unicode => "u",
}
}
}
// TODO move to ruff_python_parser?
@ -450,6 +461,14 @@ pub enum TokenKind {
Complex,
/// Token value for a string.
String,
/// Token value for the start of an f-string. This includes the `f`/`F`/`fr` prefix
/// and the opening quote(s).
FStringStart,
/// Token value that includes the portion of text inside the f-string that's not
/// part of the expression part and isn't an opening or closing brace.
FStringMiddle,
/// Token value for the end of an f-string. This includes the closing quote.
FStringEnd,
/// Token value for a IPython escape command.
EscapeCommand,
/// Token value for a comment. These are filtered out of the token stream prior to parsing.
@ -466,6 +485,8 @@ pub enum TokenKind {
EndOfFile,
/// Token value for a question mark `?`.
Question,
/// Token value for an exclamation mark `!`.
Exclamation,
/// Token value for a left parenthesis `(`.
Lpar,
/// Token value for a right parenthesis `)`.
@ -781,6 +802,9 @@ impl TokenKind {
Tok::Float { .. } => TokenKind::Float,
Tok::Complex { .. } => TokenKind::Complex,
Tok::String { .. } => TokenKind::String,
Tok::FStringStart => TokenKind::FStringStart,
Tok::FStringMiddle { .. } => TokenKind::FStringMiddle,
Tok::FStringEnd => TokenKind::FStringEnd,
Tok::IpyEscapeCommand { .. } => TokenKind::EscapeCommand,
Tok::Comment(_) => TokenKind::Comment,
Tok::Newline => TokenKind::Newline,
@ -789,6 +813,7 @@ impl TokenKind {
Tok::Dedent => TokenKind::Dedent,
Tok::EndOfFile => TokenKind::EndOfFile,
Tok::Question => TokenKind::Question,
Tok::Exclamation => TokenKind::Exclamation,
Tok::Lpar => TokenKind::Lpar,
Tok::Rpar => TokenKind::Rpar,
Tok::Lsqb => TokenKind::Lsqb,