mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-25 19:39:27 +00:00
Replace LALRPOP parser with hand-written parser (#10036)
(Supersedes #9152, authored by @LaBatata101) ## Summary This PR replaces the current parser generated from LALRPOP to a hand-written recursive descent parser. It also updates the grammar for [PEP 646](https://peps.python.org/pep-0646/) so that the parser outputs the correct AST. For example, in `data[*x]`, the index expression is now a tuple with a single starred expression instead of just a starred expression. Beyond the performance improvements, the parser is also error resilient and can provide better error messages. The behavior as seen by any downstream tools isn't changed. That is, the linter and formatter can still assume that the parser will _stop_ at the first syntax error. This will be updated in the following months. For more details about the change here, refer to the PR corresponding to the individual commits and the release blog post. ## Test Plan Write _lots_ and _lots_ of tests for both valid and invalid syntax and verify the output. ## Acknowledgements - @MichaReiser for reviewing 100+ parser PRs and continuously providing guidance throughout the project - @LaBatata101 for initiating the transition to a hand-written parser in #9152 - @addisoncrump for implementing the fuzzer which helped [catch](https://github.com/astral-sh/ruff/pull/10903) [a](https://github.com/astral-sh/ruff/pull/10910) [lot](https://github.com/astral-sh/ruff/pull/10966) [of](https://github.com/astral-sh/ruff/pull/10896) [bugs](https://github.com/astral-sh/ruff/pull/10877) --------- Co-authored-by: Victor Hugo Gomes <labatata101@linuxmail.org> Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
Dhruv Manilawala
GitHub
parent
e09180b1df
commit
13ffb5bc19
852 changed files with 112948 additions and 103620 deletions
2589
crates/ruff_python_parser/src/parser/expression.rs
Normal file
2589
crates/ruff_python_parser/src/parser/expression.rs
Normal file
File diff suppressed because it is too large
Load diff
45
crates/ruff_python_parser/src/parser/helpers.rs
Normal file
45
crates/ruff_python_parser/src/parser/helpers.rs
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
use ruff_python_ast::{self as ast, CmpOp, Expr, ExprContext};
|
||||
|
||||
use crate::TokenKind;
|
||||
|
||||
/// Set the `ctx` for `Expr::Id`, `Expr::Attribute`, `Expr::Subscript`, `Expr::Starred`,
|
||||
/// `Expr::Tuple` and `Expr::List`. If `expr` is either `Expr::Tuple` or `Expr::List`,
|
||||
/// recursively sets the `ctx` for their elements.
|
||||
pub(super) fn set_expr_ctx(expr: &mut Expr, new_ctx: ExprContext) {
|
||||
match expr {
|
||||
Expr::Name(ast::ExprName { ctx, .. })
|
||||
| Expr::Attribute(ast::ExprAttribute { ctx, .. })
|
||||
| Expr::Subscript(ast::ExprSubscript { ctx, .. }) => *ctx = new_ctx,
|
||||
Expr::Starred(ast::ExprStarred { value, ctx, .. }) => {
|
||||
*ctx = new_ctx;
|
||||
set_expr_ctx(value, new_ctx);
|
||||
}
|
||||
Expr::UnaryOp(ast::ExprUnaryOp { operand, .. }) => {
|
||||
set_expr_ctx(operand, new_ctx);
|
||||
}
|
||||
Expr::List(ast::ExprList { elts, ctx, .. })
|
||||
| Expr::Tuple(ast::ExprTuple { elts, ctx, .. }) => {
|
||||
*ctx = new_ctx;
|
||||
elts.iter_mut()
|
||||
.for_each(|element| set_expr_ctx(element, new_ctx));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a [`TokenKind`] array of size 2 to its correspondent [`CmpOp`].
|
||||
pub(super) fn token_kind_to_cmp_op(kind: [TokenKind; 2]) -> Result<CmpOp, ()> {
|
||||
Ok(match kind {
|
||||
[TokenKind::Is, TokenKind::Not] => CmpOp::IsNot,
|
||||
[TokenKind::Is, _] => CmpOp::Is,
|
||||
[TokenKind::Not, TokenKind::In] => CmpOp::NotIn,
|
||||
[TokenKind::In, _] => CmpOp::In,
|
||||
[TokenKind::EqEqual, _] => CmpOp::Eq,
|
||||
[TokenKind::NotEqual, _] => CmpOp::NotEq,
|
||||
[TokenKind::Less, _] => CmpOp::Lt,
|
||||
[TokenKind::LessEqual, _] => CmpOp::LtE,
|
||||
[TokenKind::Greater, _] => CmpOp::Gt,
|
||||
[TokenKind::GreaterEqual, _] => CmpOp::GtE,
|
||||
_ => return Err(()),
|
||||
})
|
||||
}
|
||||
1288
crates/ruff_python_parser/src/parser/mod.rs
Normal file
1288
crates/ruff_python_parser/src/parser/mod.rs
Normal file
File diff suppressed because it is too large
Load diff
752
crates/ruff_python_parser/src/parser/pattern.rs
Normal file
752
crates/ruff_python_parser/src/parser/pattern.rs
Normal file
|
|
@ -0,0 +1,752 @@
|
|||
use ruff_python_ast::{self as ast, Expr, ExprContext, Number, Operator, Pattern, Singleton};
|
||||
use ruff_text_size::{Ranged, TextSize};
|
||||
|
||||
use crate::parser::progress::ParserProgress;
|
||||
use crate::parser::{recovery, Parser, RecoveryContextKind, SequenceMatchPatternParentheses};
|
||||
use crate::token_set::TokenSet;
|
||||
use crate::{ParseErrorType, Tok, TokenKind};
|
||||
|
||||
/// The set of tokens that can start a literal pattern.
|
||||
const LITERAL_PATTERN_START_SET: TokenSet = TokenSet::new([
|
||||
TokenKind::None,
|
||||
TokenKind::True,
|
||||
TokenKind::False,
|
||||
TokenKind::String,
|
||||
TokenKind::Int,
|
||||
TokenKind::Float,
|
||||
TokenKind::Complex,
|
||||
TokenKind::Minus, // Unary minus
|
||||
]);
|
||||
|
||||
/// The set of tokens that can start a pattern.
|
||||
const PATTERN_START_SET: TokenSet = TokenSet::new([
|
||||
// Star pattern
|
||||
TokenKind::Star,
|
||||
// Capture pattern
|
||||
// Wildcard pattern ('_' is a name token)
|
||||
// Value pattern (name or attribute)
|
||||
// Class pattern
|
||||
TokenKind::Name,
|
||||
// Group pattern
|
||||
TokenKind::Lpar,
|
||||
// Sequence pattern
|
||||
TokenKind::Lsqb,
|
||||
// Mapping pattern
|
||||
TokenKind::Lbrace,
|
||||
])
|
||||
.union(LITERAL_PATTERN_START_SET);
|
||||
|
||||
/// The set of tokens that can start a mapping pattern.
|
||||
const MAPPING_PATTERN_START_SET: TokenSet = TokenSet::new([
|
||||
// Double star pattern
|
||||
TokenKind::DoubleStar,
|
||||
// Value pattern
|
||||
TokenKind::Name,
|
||||
])
|
||||
.union(LITERAL_PATTERN_START_SET);
|
||||
|
||||
impl<'src> Parser<'src> {
|
||||
/// Returns `true` if the current token is a valid start of a pattern.
|
||||
pub(super) fn at_pattern_start(&self) -> bool {
|
||||
self.at_ts(PATTERN_START_SET)
|
||||
}
|
||||
|
||||
/// Returns `true` if the current token is a valid start of a mapping pattern.
|
||||
pub(super) fn at_mapping_pattern_start(&self) -> bool {
|
||||
self.at_ts(MAPPING_PATTERN_START_SET)
|
||||
}
|
||||
|
||||
/// Entry point to start parsing a pattern.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns>
|
||||
pub(super) fn parse_match_patterns(&mut self) -> Pattern {
|
||||
let start = self.node_start();
|
||||
|
||||
// We don't yet know if it's a sequence pattern or a single pattern, so
|
||||
// we need to allow star pattern here.
|
||||
let pattern = self.parse_match_pattern(AllowStarPattern::Yes);
|
||||
|
||||
if self.at(TokenKind::Comma) {
|
||||
Pattern::MatchSequence(self.parse_sequence_match_pattern(pattern, start, None))
|
||||
} else {
|
||||
// We know it's not a sequence pattern now, so check for star pattern usage.
|
||||
if pattern.is_match_star() {
|
||||
self.add_error(ParseErrorType::InvalidStarPatternUsage, &pattern);
|
||||
}
|
||||
pattern
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses an `or_pattern` or an `as_pattern`.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-pattern>
|
||||
fn parse_match_pattern(&mut self, allow_star_pattern: AllowStarPattern) -> Pattern {
|
||||
let start = self.node_start();
|
||||
|
||||
// We don't yet know if it's an or pattern or an as pattern, so use whatever
|
||||
// was passed in.
|
||||
let mut lhs = self.parse_match_pattern_lhs(allow_star_pattern);
|
||||
|
||||
// Or pattern
|
||||
if self.at(TokenKind::Vbar) {
|
||||
// We know it's an `or` pattern now, so check for star pattern usage.
|
||||
if lhs.is_match_star() {
|
||||
self.add_error(ParseErrorType::InvalidStarPatternUsage, &lhs);
|
||||
}
|
||||
|
||||
let mut patterns = vec![lhs];
|
||||
let mut progress = ParserProgress::default();
|
||||
|
||||
while self.eat(TokenKind::Vbar) {
|
||||
progress.assert_progressing(self);
|
||||
let pattern = self.parse_match_pattern_lhs(AllowStarPattern::No);
|
||||
patterns.push(pattern);
|
||||
}
|
||||
|
||||
lhs = Pattern::MatchOr(ast::PatternMatchOr {
|
||||
range: self.node_range(start),
|
||||
patterns,
|
||||
});
|
||||
}
|
||||
|
||||
// As pattern
|
||||
if self.eat(TokenKind::As) {
|
||||
// We know it's an `as` pattern now, so check for star pattern usage.
|
||||
if lhs.is_match_star() {
|
||||
self.add_error(ParseErrorType::InvalidStarPatternUsage, &lhs);
|
||||
}
|
||||
|
||||
let ident = self.parse_identifier();
|
||||
lhs = Pattern::MatchAs(ast::PatternMatchAs {
|
||||
range: self.node_range(start),
|
||||
name: Some(ident),
|
||||
pattern: Some(Box::new(lhs)),
|
||||
});
|
||||
}
|
||||
|
||||
lhs
|
||||
}
|
||||
|
||||
/// Parses a pattern.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-closed_pattern>
|
||||
fn parse_match_pattern_lhs(&mut self, allow_star_pattern: AllowStarPattern) -> Pattern {
|
||||
let start = self.node_start();
|
||||
|
||||
let mut lhs = match self.current_token_kind() {
|
||||
TokenKind::Lbrace => Pattern::MatchMapping(self.parse_match_pattern_mapping()),
|
||||
TokenKind::Star => {
|
||||
let star_pattern = self.parse_match_pattern_star();
|
||||
if allow_star_pattern.is_no() {
|
||||
self.add_error(ParseErrorType::InvalidStarPatternUsage, &star_pattern);
|
||||
}
|
||||
Pattern::MatchStar(star_pattern)
|
||||
}
|
||||
TokenKind::Lpar | TokenKind::Lsqb => self.parse_parenthesized_or_sequence_pattern(),
|
||||
_ => self.parse_match_pattern_literal(),
|
||||
};
|
||||
|
||||
if self.at(TokenKind::Lpar) {
|
||||
lhs = Pattern::MatchClass(self.parse_match_pattern_class(lhs, start));
|
||||
}
|
||||
|
||||
if matches!(
|
||||
self.current_token_kind(),
|
||||
TokenKind::Plus | TokenKind::Minus
|
||||
) {
|
||||
lhs = Pattern::MatchValue(self.parse_complex_literal_pattern(lhs, start));
|
||||
}
|
||||
|
||||
lhs
|
||||
}
|
||||
|
||||
/// Parses a mapping pattern.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `{` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#mapping-patterns>
|
||||
fn parse_match_pattern_mapping(&mut self) -> ast::PatternMatchMapping {
|
||||
let start = self.node_start();
|
||||
self.bump(TokenKind::Lbrace);
|
||||
|
||||
let mut keys = vec![];
|
||||
let mut patterns = vec![];
|
||||
let mut rest = None;
|
||||
|
||||
self.parse_comma_separated_list(RecoveryContextKind::MatchPatternMapping, |parser| {
|
||||
let mapping_item_start = parser.node_start();
|
||||
|
||||
if parser.eat(TokenKind::DoubleStar) {
|
||||
let identifier = parser.parse_identifier();
|
||||
if rest.is_some() {
|
||||
parser.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Only one double star pattern is allowed".to_string(),
|
||||
),
|
||||
parser.node_range(mapping_item_start),
|
||||
);
|
||||
}
|
||||
// TODO(dhruvmanila): It's not possible to retain multiple double starred
|
||||
// patterns because of the way the mapping node is represented in the grammar.
|
||||
// The last value will always win. Update the AST representation.
|
||||
// See: https://github.com/astral-sh/ruff/pull/10477#discussion_r1535143536
|
||||
rest = Some(identifier);
|
||||
} else {
|
||||
let key = match parser.parse_match_pattern_lhs(AllowStarPattern::No) {
|
||||
Pattern::MatchValue(ast::PatternMatchValue { value, .. }) => *value,
|
||||
Pattern::MatchSingleton(ast::PatternMatchSingleton { value, range }) => {
|
||||
match value {
|
||||
Singleton::None => Expr::NoneLiteral(ast::ExprNoneLiteral { range }),
|
||||
Singleton::True => {
|
||||
Expr::BooleanLiteral(ast::ExprBooleanLiteral { value: true, range })
|
||||
}
|
||||
Singleton::False => Expr::BooleanLiteral(ast::ExprBooleanLiteral {
|
||||
value: false,
|
||||
range,
|
||||
}),
|
||||
}
|
||||
}
|
||||
pattern => {
|
||||
parser.add_error(
|
||||
ParseErrorType::OtherError("Invalid mapping pattern key".to_string()),
|
||||
&pattern,
|
||||
);
|
||||
recovery::pattern_to_expr(pattern)
|
||||
}
|
||||
};
|
||||
keys.push(key);
|
||||
|
||||
parser.expect(TokenKind::Colon);
|
||||
|
||||
patterns.push(parser.parse_match_pattern(AllowStarPattern::No));
|
||||
|
||||
if rest.is_some() {
|
||||
parser.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Pattern cannot follow a double star pattern".to_string(),
|
||||
),
|
||||
parser.node_range(mapping_item_start),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
self.expect(TokenKind::Rbrace);
|
||||
|
||||
ast::PatternMatchMapping {
|
||||
range: self.node_range(start),
|
||||
keys,
|
||||
patterns,
|
||||
rest,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a star pattern.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `*` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-star_pattern>
|
||||
fn parse_match_pattern_star(&mut self) -> ast::PatternMatchStar {
|
||||
let start = self.node_start();
|
||||
self.bump(TokenKind::Star);
|
||||
|
||||
let ident = self.parse_identifier();
|
||||
|
||||
ast::PatternMatchStar {
|
||||
range: self.node_range(start),
|
||||
name: if ident.is_valid() && ident.id == "_" {
|
||||
None
|
||||
} else {
|
||||
Some(ident)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a parenthesized pattern or a sequence pattern.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `(` or `[` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#sequence-patterns>
|
||||
fn parse_parenthesized_or_sequence_pattern(&mut self) -> Pattern {
|
||||
let start = self.node_start();
|
||||
let parentheses = if self.eat(TokenKind::Lpar) {
|
||||
SequenceMatchPatternParentheses::Tuple
|
||||
} else {
|
||||
self.bump(TokenKind::Lsqb);
|
||||
SequenceMatchPatternParentheses::List
|
||||
};
|
||||
|
||||
if matches!(
|
||||
self.current_token_kind(),
|
||||
TokenKind::Newline | TokenKind::Colon
|
||||
) {
|
||||
// TODO(dhruvmanila): This recovery isn't possible currently because
|
||||
// of the soft keyword transformer. If there's a missing closing
|
||||
// parenthesis, it'll consider `case` a name token instead.
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError(format!(
|
||||
"Missing '{closing}'",
|
||||
closing = if parentheses.is_list() { "]" } else { ")" }
|
||||
)),
|
||||
self.current_token_range(),
|
||||
);
|
||||
}
|
||||
|
||||
if self.eat(parentheses.closing_kind()) {
|
||||
return Pattern::MatchSequence(ast::PatternMatchSequence {
|
||||
patterns: vec![],
|
||||
range: self.node_range(start),
|
||||
});
|
||||
}
|
||||
|
||||
let mut pattern = self.parse_match_pattern(AllowStarPattern::Yes);
|
||||
|
||||
if parentheses.is_list() || self.at(TokenKind::Comma) {
|
||||
pattern = Pattern::MatchSequence(self.parse_sequence_match_pattern(
|
||||
pattern,
|
||||
start,
|
||||
Some(parentheses),
|
||||
));
|
||||
} else {
|
||||
self.expect(parentheses.closing_kind());
|
||||
}
|
||||
|
||||
pattern
|
||||
}
|
||||
|
||||
/// Parses the rest of a sequence pattern, given the first element.
|
||||
///
|
||||
/// If the `parentheses` is `None`, it is an [open sequence pattern].
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#sequence-patterns>
|
||||
///
|
||||
/// [open sequence pattern]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-open_sequence_pattern
|
||||
fn parse_sequence_match_pattern(
|
||||
&mut self,
|
||||
first_element: Pattern,
|
||||
start: TextSize,
|
||||
parentheses: Option<SequenceMatchPatternParentheses>,
|
||||
) -> ast::PatternMatchSequence {
|
||||
if parentheses.is_some_and(|parentheses| {
|
||||
self.at(parentheses.closing_kind()) || self.peek() == parentheses.closing_kind()
|
||||
}) {
|
||||
// The comma is optional if it is a single-element sequence
|
||||
self.eat(TokenKind::Comma);
|
||||
} else {
|
||||
self.expect(TokenKind::Comma);
|
||||
}
|
||||
|
||||
let mut patterns = vec![first_element];
|
||||
|
||||
self.parse_comma_separated_list(
|
||||
RecoveryContextKind::SequenceMatchPattern(parentheses),
|
||||
|parser| patterns.push(parser.parse_match_pattern(AllowStarPattern::Yes)),
|
||||
);
|
||||
|
||||
if let Some(parentheses) = parentheses {
|
||||
self.expect(parentheses.closing_kind());
|
||||
}
|
||||
|
||||
ast::PatternMatchSequence {
|
||||
range: self.node_range(start),
|
||||
patterns,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a literal pattern.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-literal_pattern>
|
||||
fn parse_match_pattern_literal(&mut self) -> Pattern {
|
||||
let start = self.node_start();
|
||||
match self.current_token_kind() {
|
||||
TokenKind::None => {
|
||||
self.bump(TokenKind::None);
|
||||
Pattern::MatchSingleton(ast::PatternMatchSingleton {
|
||||
value: Singleton::None,
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
TokenKind::True => {
|
||||
self.bump(TokenKind::True);
|
||||
Pattern::MatchSingleton(ast::PatternMatchSingleton {
|
||||
value: Singleton::True,
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
TokenKind::False => {
|
||||
self.bump(TokenKind::False);
|
||||
Pattern::MatchSingleton(ast::PatternMatchSingleton {
|
||||
value: Singleton::False,
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
TokenKind::String | TokenKind::FStringStart => {
|
||||
let str = self.parse_strings();
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(str),
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
TokenKind::Complex => {
|
||||
let (Tok::Complex { real, imag }, _) = self.bump(TokenKind::Complex) else {
|
||||
unreachable!()
|
||||
};
|
||||
let range = self.node_range(start);
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
|
||||
value: Number::Complex { real, imag },
|
||||
range,
|
||||
})),
|
||||
range,
|
||||
})
|
||||
}
|
||||
TokenKind::Int => {
|
||||
let (Tok::Int { value }, _) = self.bump(TokenKind::Int) else {
|
||||
unreachable!()
|
||||
};
|
||||
let range = self.node_range(start);
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
|
||||
value: Number::Int(value),
|
||||
range,
|
||||
})),
|
||||
range,
|
||||
})
|
||||
}
|
||||
TokenKind::Float => {
|
||||
let (Tok::Float { value }, _) = self.bump(TokenKind::Float) else {
|
||||
unreachable!()
|
||||
};
|
||||
let range = self.node_range(start);
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
|
||||
value: Number::Float(value),
|
||||
range,
|
||||
})),
|
||||
range,
|
||||
})
|
||||
}
|
||||
TokenKind::Name if self.peek() == TokenKind::Dot => {
|
||||
let (Tok::Name { name }, _) = self.bump(TokenKind::Name) else {
|
||||
unreachable!()
|
||||
};
|
||||
let id = Expr::Name(ast::ExprName {
|
||||
id: name.to_string(),
|
||||
ctx: ExprContext::Load,
|
||||
range: self.node_range(start),
|
||||
});
|
||||
|
||||
let attribute = self.parse_attr_expr_for_match_pattern(id, start);
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(attribute),
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
TokenKind::Name => {
|
||||
let (Tok::Name { name }, _) = self.bump(TokenKind::Name) else {
|
||||
unreachable!()
|
||||
};
|
||||
let range = self.node_range(start);
|
||||
|
||||
// test_ok match_as_pattern
|
||||
// match foo:
|
||||
// case foo_bar: ...
|
||||
// case _: ...
|
||||
Pattern::MatchAs(ast::PatternMatchAs {
|
||||
range,
|
||||
pattern: None,
|
||||
name: if &*name == "_" {
|
||||
None
|
||||
} else {
|
||||
Some(ast::Identifier {
|
||||
id: name.to_string(),
|
||||
range,
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
// The `+` is only for better error recovery.
|
||||
TokenKind::Minus | TokenKind::Plus
|
||||
if matches!(
|
||||
self.peek(),
|
||||
TokenKind::Int | TokenKind::Float | TokenKind::Complex
|
||||
) =>
|
||||
{
|
||||
let unary_expr = self.parse_unary_expression();
|
||||
|
||||
if unary_expr.op.is_u_add() {
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Unary '+' is not allowed as a literal pattern".to_string(),
|
||||
),
|
||||
&unary_expr,
|
||||
);
|
||||
}
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
value: Box::new(Expr::UnaryOp(unary_expr)),
|
||||
range: self.node_range(start),
|
||||
})
|
||||
}
|
||||
kind => {
|
||||
// Upon encountering an unexpected token, return a `Pattern::MatchValue` containing
|
||||
// an empty `Expr::Name`.
|
||||
let invalid_node = if kind.is_keyword() {
|
||||
Expr::Name(self.parse_name())
|
||||
} else {
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError("Expected a pattern".to_string()),
|
||||
self.current_token_range(),
|
||||
);
|
||||
Expr::Name(ast::ExprName {
|
||||
range: self.missing_node_range(),
|
||||
id: String::new(),
|
||||
ctx: ExprContext::Invalid,
|
||||
})
|
||||
};
|
||||
|
||||
Pattern::MatchValue(ast::PatternMatchValue {
|
||||
range: invalid_node.range(),
|
||||
value: Box::new(invalid_node),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a complex literal pattern, given the `lhs` pattern and the `start`
|
||||
/// position of the pattern.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `+` or `-` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#literal-patterns>
|
||||
fn parse_complex_literal_pattern(
|
||||
&mut self,
|
||||
lhs: Pattern,
|
||||
start: TextSize,
|
||||
) -> ast::PatternMatchValue {
|
||||
let operator = if self.eat(TokenKind::Plus) {
|
||||
Operator::Add
|
||||
} else {
|
||||
self.bump(TokenKind::Minus);
|
||||
Operator::Sub
|
||||
};
|
||||
|
||||
let lhs_value = if let Pattern::MatchValue(lhs) = lhs {
|
||||
if !is_real_number(&lhs.value) {
|
||||
self.add_error(ParseErrorType::ExpectedRealNumber, &lhs);
|
||||
}
|
||||
lhs.value
|
||||
} else {
|
||||
self.add_error(ParseErrorType::ExpectedRealNumber, &lhs);
|
||||
Box::new(recovery::pattern_to_expr(lhs))
|
||||
};
|
||||
|
||||
let rhs_pattern = self.parse_match_pattern_lhs(AllowStarPattern::No);
|
||||
let rhs_value = if let Pattern::MatchValue(rhs) = rhs_pattern {
|
||||
if !is_complex_number(&rhs.value) {
|
||||
self.add_error(ParseErrorType::ExpectedImaginaryNumber, &rhs);
|
||||
}
|
||||
rhs.value
|
||||
} else {
|
||||
self.add_error(ParseErrorType::ExpectedImaginaryNumber, &rhs_pattern);
|
||||
Box::new(recovery::pattern_to_expr(rhs_pattern))
|
||||
};
|
||||
|
||||
let range = self.node_range(start);
|
||||
|
||||
ast::PatternMatchValue {
|
||||
value: Box::new(Expr::BinOp(ast::ExprBinOp {
|
||||
left: lhs_value,
|
||||
op: operator,
|
||||
right: rhs_value,
|
||||
range,
|
||||
})),
|
||||
range,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses an attribute expression until the current token is not a `.`.
|
||||
fn parse_attr_expr_for_match_pattern(&mut self, mut lhs: Expr, start: TextSize) -> Expr {
|
||||
while self.current_token_kind() == TokenKind::Dot {
|
||||
lhs = Expr::Attribute(self.parse_attribute_expression(lhs, start));
|
||||
}
|
||||
|
||||
lhs
|
||||
}
|
||||
|
||||
/// Parses the [pattern arguments] in a class pattern.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `(` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#class-patterns>
|
||||
///
|
||||
/// [pattern arguments]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-pattern_arguments
|
||||
fn parse_match_pattern_class(
|
||||
&mut self,
|
||||
cls: Pattern,
|
||||
start: TextSize,
|
||||
) -> ast::PatternMatchClass {
|
||||
let arguments_start = self.node_start();
|
||||
|
||||
let cls = match cls {
|
||||
Pattern::MatchAs(ast::PatternMatchAs {
|
||||
pattern: None,
|
||||
name: Some(ident),
|
||||
..
|
||||
}) => {
|
||||
if ident.is_valid() {
|
||||
Box::new(Expr::Name(ast::ExprName {
|
||||
range: ident.range(),
|
||||
id: ident.id,
|
||||
ctx: ExprContext::Load,
|
||||
}))
|
||||
} else {
|
||||
Box::new(Expr::Name(ast::ExprName {
|
||||
range: ident.range(),
|
||||
id: String::new(),
|
||||
ctx: ExprContext::Invalid,
|
||||
}))
|
||||
}
|
||||
}
|
||||
Pattern::MatchValue(ast::PatternMatchValue { value, .. })
|
||||
if matches!(&*value, Expr::Attribute(_)) =>
|
||||
{
|
||||
value
|
||||
}
|
||||
pattern => {
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError("Invalid value for a class pattern".to_string()),
|
||||
&pattern,
|
||||
);
|
||||
Box::new(recovery::pattern_to_expr(pattern))
|
||||
}
|
||||
};
|
||||
|
||||
self.bump(TokenKind::Lpar);
|
||||
|
||||
let mut patterns = vec![];
|
||||
let mut keywords = vec![];
|
||||
let mut has_seen_pattern = false;
|
||||
let mut has_seen_keyword_pattern = false;
|
||||
|
||||
self.parse_comma_separated_list(
|
||||
RecoveryContextKind::MatchPatternClassArguments,
|
||||
|parser| {
|
||||
let pattern_start = parser.node_start();
|
||||
let pattern = parser.parse_match_pattern(AllowStarPattern::No);
|
||||
|
||||
if parser.eat(TokenKind::Equal) {
|
||||
has_seen_pattern = false;
|
||||
has_seen_keyword_pattern = true;
|
||||
|
||||
let key = if let Pattern::MatchAs(ast::PatternMatchAs {
|
||||
pattern: None,
|
||||
name: Some(name),
|
||||
..
|
||||
}) = pattern
|
||||
{
|
||||
name
|
||||
} else {
|
||||
parser.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Expected an identifier for the keyword pattern".to_string(),
|
||||
),
|
||||
&pattern,
|
||||
);
|
||||
ast::Identifier {
|
||||
id: String::new(),
|
||||
range: parser.missing_node_range(),
|
||||
}
|
||||
};
|
||||
|
||||
let value_pattern = parser.parse_match_pattern(AllowStarPattern::No);
|
||||
|
||||
keywords.push(ast::PatternKeyword {
|
||||
attr: key,
|
||||
pattern: value_pattern,
|
||||
range: parser.node_range(pattern_start),
|
||||
});
|
||||
} else {
|
||||
has_seen_pattern = true;
|
||||
patterns.push(pattern);
|
||||
}
|
||||
|
||||
if has_seen_keyword_pattern && has_seen_pattern {
|
||||
parser.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Positional patterns cannot follow keyword patterns".to_string(),
|
||||
),
|
||||
parser.node_range(pattern_start),
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
self.expect(TokenKind::Rpar);
|
||||
|
||||
ast::PatternMatchClass {
|
||||
cls,
|
||||
arguments: ast::PatternArguments {
|
||||
patterns,
|
||||
keywords,
|
||||
range: self.node_range(arguments_start),
|
||||
},
|
||||
range: self.node_range(start),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum AllowStarPattern {
|
||||
Yes,
|
||||
No,
|
||||
}
|
||||
|
||||
impl AllowStarPattern {
|
||||
const fn is_no(self) -> bool {
|
||||
matches!(self, AllowStarPattern::No)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the given expression is a real number literal or a unary
|
||||
/// addition or subtraction of a real number literal.
|
||||
const fn is_real_number(expr: &Expr) -> bool {
|
||||
match expr {
|
||||
Expr::NumberLiteral(ast::ExprNumberLiteral {
|
||||
value: ast::Number::Int(_) | ast::Number::Float(_),
|
||||
..
|
||||
}) => true,
|
||||
Expr::UnaryOp(ast::ExprUnaryOp {
|
||||
op: ast::UnaryOp::UAdd | ast::UnaryOp::USub,
|
||||
operand,
|
||||
..
|
||||
}) => is_real_number(operand),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the given expression is a complex number literal.
|
||||
const fn is_complex_number(expr: &Expr) -> bool {
|
||||
matches!(
|
||||
expr,
|
||||
Expr::NumberLiteral(ast::ExprNumberLiteral {
|
||||
value: ast::Number::Complex { .. },
|
||||
..
|
||||
})
|
||||
)
|
||||
}
|
||||
46
crates/ruff_python_parser/src/parser/progress.rs
Normal file
46
crates/ruff_python_parser/src/parser/progress.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
use crate::parser::Parser;
|
||||
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub(super) struct TokenId(u32);
|
||||
|
||||
impl TokenId {
|
||||
/// Increments the value of the token ID.
|
||||
pub(super) fn increment(&mut self) {
|
||||
// It's fine to just wrap around because the main purpose is to check whether
|
||||
// the previous token ID is different from the current token ID.
|
||||
self.0 = self.0.wrapping_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Captures the progress of the parser and allows to test if the parsing is still making progress
|
||||
#[derive(Debug, Copy, Clone, Default)]
|
||||
pub(super) struct ParserProgress(Option<TokenId>);
|
||||
|
||||
impl ParserProgress {
|
||||
/// Returns true if the parser has passed this position
|
||||
#[inline]
|
||||
fn has_progressed(self, p: &Parser) -> bool {
|
||||
match self.0 {
|
||||
None => true,
|
||||
Some(prev_token_id) => prev_token_id != p.current_token_id(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Asserts that the parsing is still making progress.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the parser hasn't progressed since the last call.
|
||||
#[inline]
|
||||
pub(super) fn assert_progressing(&mut self, p: &Parser) {
|
||||
assert!(
|
||||
self.has_progressed(p),
|
||||
"The parser is no longer progressing. Stuck at '{}' {:?}:{:?}",
|
||||
p.src_text(p.current_token_range()),
|
||||
p.current_token_kind(),
|
||||
p.current_token_range(),
|
||||
);
|
||||
|
||||
self.0 = Some(p.current_token_id());
|
||||
}
|
||||
}
|
||||
168
crates/ruff_python_parser/src/parser/recovery.rs
Normal file
168
crates/ruff_python_parser/src/parser/recovery.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
use ruff_python_ast::{self as ast, Expr, ExprContext, Pattern};
|
||||
use ruff_text_size::{Ranged, TextLen, TextRange};
|
||||
|
||||
/// Convert the given [`Pattern`] to an [`Expr`].
|
||||
///
|
||||
/// This is used to convert an invalid use of pattern to their equivalent expression
|
||||
/// to preserve the structure of the pattern.
|
||||
///
|
||||
/// The conversion is done as follows:
|
||||
/// - `PatternMatchSingleton`: Boolean and None literals
|
||||
/// - `PatternMatchValue`: The value itself
|
||||
/// - `PatternMatchSequence`: List literal
|
||||
/// - `PatternMatchMapping`: Dictionary literal
|
||||
/// - `PatternMatchClass`: Call expression
|
||||
/// - `PatternMatchStar`: Starred expression
|
||||
/// - `PatternMatchAs`: The pattern itself or the name
|
||||
/// - `PatternMatchOr`: Binary expression with `|` operator
|
||||
///
|
||||
/// Note that the sequence pattern is always converted to a list literal even
|
||||
/// if it was surrounded by parentheses.
|
||||
///
|
||||
/// # Note
|
||||
///
|
||||
/// This function returns an invalid [`ast::ExprName`] if the given pattern is a [`Pattern::MatchAs`]
|
||||
/// with both the pattern and name present. This is because it cannot be converted to an expression
|
||||
/// without dropping one of them as there's no way to represent `x as y` as a valid expression.
|
||||
pub(super) fn pattern_to_expr(pattern: Pattern) -> Expr {
|
||||
match pattern {
|
||||
Pattern::MatchSingleton(ast::PatternMatchSingleton { range, value }) => match value {
|
||||
ast::Singleton::True => {
|
||||
Expr::BooleanLiteral(ast::ExprBooleanLiteral { value: true, range })
|
||||
}
|
||||
ast::Singleton::False => Expr::BooleanLiteral(ast::ExprBooleanLiteral {
|
||||
value: false,
|
||||
range,
|
||||
}),
|
||||
ast::Singleton::None => Expr::NoneLiteral(ast::ExprNoneLiteral { range }),
|
||||
},
|
||||
Pattern::MatchValue(ast::PatternMatchValue { value, .. }) => *value,
|
||||
// We don't know which kind of sequence this is: `case [1, 2]:` or `case (1, 2):`.
|
||||
Pattern::MatchSequence(ast::PatternMatchSequence { range, patterns }) => {
|
||||
Expr::List(ast::ExprList {
|
||||
elts: patterns.into_iter().map(pattern_to_expr).collect(),
|
||||
ctx: ExprContext::Store,
|
||||
range,
|
||||
})
|
||||
}
|
||||
Pattern::MatchMapping(ast::PatternMatchMapping {
|
||||
range,
|
||||
keys,
|
||||
patterns,
|
||||
rest,
|
||||
}) => {
|
||||
let mut keys = keys.into_iter().map(Option::Some).collect::<Vec<_>>();
|
||||
let mut values = patterns
|
||||
.into_iter()
|
||||
.map(pattern_to_expr)
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(rest) = rest {
|
||||
keys.push(None);
|
||||
values.push(Expr::Name(ast::ExprName {
|
||||
range: rest.range,
|
||||
id: rest.id,
|
||||
ctx: ExprContext::Store,
|
||||
}));
|
||||
}
|
||||
Expr::Dict(ast::ExprDict {
|
||||
range,
|
||||
keys,
|
||||
values,
|
||||
})
|
||||
}
|
||||
Pattern::MatchClass(ast::PatternMatchClass {
|
||||
range,
|
||||
cls,
|
||||
arguments,
|
||||
}) => Expr::Call(ast::ExprCall {
|
||||
range,
|
||||
func: cls,
|
||||
arguments: ast::Arguments {
|
||||
range: arguments.range,
|
||||
args: arguments
|
||||
.patterns
|
||||
.into_iter()
|
||||
.map(pattern_to_expr)
|
||||
.collect(),
|
||||
keywords: arguments
|
||||
.keywords
|
||||
.into_iter()
|
||||
.map(|keyword_pattern| ast::Keyword {
|
||||
range: keyword_pattern.range,
|
||||
arg: Some(keyword_pattern.attr),
|
||||
value: pattern_to_expr(keyword_pattern.pattern),
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
}),
|
||||
Pattern::MatchStar(ast::PatternMatchStar { range, name }) => {
|
||||
if let Some(name) = name {
|
||||
Expr::Starred(ast::ExprStarred {
|
||||
range,
|
||||
value: Box::new(Expr::Name(ast::ExprName {
|
||||
range: name.range,
|
||||
id: name.id,
|
||||
ctx: ExprContext::Store,
|
||||
})),
|
||||
ctx: ExprContext::Store,
|
||||
})
|
||||
} else {
|
||||
Expr::Starred(ast::ExprStarred {
|
||||
range,
|
||||
value: Box::new(Expr::Name(ast::ExprName {
|
||||
range: TextRange::new(range.end() - "_".text_len(), range.end()),
|
||||
id: "_".to_string(),
|
||||
ctx: ExprContext::Store,
|
||||
})),
|
||||
ctx: ExprContext::Store,
|
||||
})
|
||||
}
|
||||
}
|
||||
Pattern::MatchAs(ast::PatternMatchAs {
|
||||
range,
|
||||
pattern,
|
||||
name,
|
||||
}) => match (pattern, name) {
|
||||
(Some(_), Some(_)) => Expr::Name(ast::ExprName {
|
||||
range,
|
||||
id: String::new(),
|
||||
ctx: ExprContext::Invalid,
|
||||
}),
|
||||
(Some(pattern), None) => pattern_to_expr(*pattern),
|
||||
(None, Some(name)) => Expr::Name(ast::ExprName {
|
||||
range: name.range,
|
||||
id: name.id,
|
||||
ctx: ExprContext::Store,
|
||||
}),
|
||||
(None, None) => Expr::Name(ast::ExprName {
|
||||
range,
|
||||
id: "_".to_string(),
|
||||
ctx: ExprContext::Store,
|
||||
}),
|
||||
},
|
||||
Pattern::MatchOr(ast::PatternMatchOr { patterns, .. }) => {
|
||||
let to_bin_expr = |left: Pattern, right: Pattern| ast::ExprBinOp {
|
||||
range: TextRange::new(left.start(), right.end()),
|
||||
left: Box::new(pattern_to_expr(left)),
|
||||
op: ast::Operator::BitOr,
|
||||
right: Box::new(pattern_to_expr(right)),
|
||||
};
|
||||
|
||||
let mut iter = patterns.into_iter();
|
||||
|
||||
match (iter.next(), iter.next()) {
|
||||
(Some(left), Some(right)) => {
|
||||
Expr::BinOp(iter.fold(to_bin_expr(left, right), |expr_bin_op, pattern| {
|
||||
ast::ExprBinOp {
|
||||
range: TextRange::new(expr_bin_op.start(), pattern.end()),
|
||||
left: Box::new(Expr::BinOp(expr_bin_op)),
|
||||
op: ast::Operator::BitOr,
|
||||
right: Box::new(pattern_to_expr(pattern)),
|
||||
}
|
||||
}))
|
||||
}
|
||||
_ => unreachable!("Or patterns can only be formed with at least two patterns."),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: error
|
||||
---
|
||||
ParseError {
|
||||
error: UnexpectedExpressionToken,
|
||||
location: 6..12,
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: error
|
||||
---
|
||||
ParseError {
|
||||
error: UnexpectedExpressionToken,
|
||||
location: 7..13,
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: error
|
||||
---
|
||||
ParseError {
|
||||
error: UnexpectedExpressionToken,
|
||||
location: 7..13,
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: expr
|
||||
---
|
||||
Name(
|
||||
ExprName {
|
||||
range: 0..5,
|
||||
id: "first",
|
||||
ctx: Load,
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,399 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: parse_ast
|
||||
---
|
||||
Module(
|
||||
ModModule {
|
||||
range: 0..929,
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 21..42,
|
||||
value: BinOp(
|
||||
ExprBinOp {
|
||||
range: 27..40,
|
||||
left: Name(
|
||||
ExprName {
|
||||
range: 27..28,
|
||||
id: "a",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
op: Mod,
|
||||
right: Name(
|
||||
ExprName {
|
||||
range: 39..40,
|
||||
id: "b",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 66..73,
|
||||
kind: Help2,
|
||||
value: "a.foo",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 74..80,
|
||||
kind: Help,
|
||||
value: "a.foo",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 81..88,
|
||||
kind: Help,
|
||||
value: "a.foo",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 89..100,
|
||||
kind: Help2,
|
||||
value: "a.foo()",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 115..128,
|
||||
kind: Magic,
|
||||
value: "timeit a = b",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 129..147,
|
||||
kind: Magic,
|
||||
value: "timeit foo(b) % 3",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 148..176,
|
||||
kind: Magic,
|
||||
value: "alias showPath pwd && ls -a",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 177..205,
|
||||
kind: Magic,
|
||||
value: "timeit a = foo(b); b = 2",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 206..226,
|
||||
kind: Magic,
|
||||
value: "matplotlib --inline",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 227..253,
|
||||
kind: Magic,
|
||||
value: "matplotlib --inline",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 277..309,
|
||||
kind: Shell,
|
||||
value: "pwd && ls -a | sed 's/^/\\ /'",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 310..347,
|
||||
kind: Shell,
|
||||
value: "pwd && ls -a | sed 's/^/\\\\ /'",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 348..393,
|
||||
kind: ShCap,
|
||||
value: "cd /Users/foo/Library/Application\\ Support/",
|
||||
},
|
||||
),
|
||||
FunctionDef(
|
||||
StmtFunctionDef {
|
||||
range: 566..626,
|
||||
is_async: false,
|
||||
decorator_list: [],
|
||||
name: Identifier {
|
||||
id: "foo",
|
||||
range: 570..573,
|
||||
},
|
||||
type_params: None,
|
||||
parameters: Parameters {
|
||||
range: 573..575,
|
||||
posonlyargs: [],
|
||||
args: [],
|
||||
vararg: None,
|
||||
kwonlyargs: [],
|
||||
kwarg: None,
|
||||
},
|
||||
returns: None,
|
||||
body: [
|
||||
Return(
|
||||
StmtReturn {
|
||||
range: 581..626,
|
||||
value: Some(
|
||||
Compare(
|
||||
ExprCompare {
|
||||
range: 598..620,
|
||||
left: Name(
|
||||
ExprName {
|
||||
range: 598..599,
|
||||
id: "a",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
ops: [
|
||||
NotEq,
|
||||
],
|
||||
comparators: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 619..620,
|
||||
id: "b",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 656..664,
|
||||
kind: Paren,
|
||||
value: "foo 1 2",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 665..673,
|
||||
kind: Quote2,
|
||||
value: "foo 1 2",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 674..682,
|
||||
kind: Quote,
|
||||
value: "foo 1 2",
|
||||
},
|
||||
),
|
||||
For(
|
||||
StmtFor {
|
||||
range: 711..737,
|
||||
is_async: false,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 715..716,
|
||||
id: "a",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 720..728,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 720..725,
|
||||
id: "range",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
arguments: Arguments {
|
||||
range: 725..728,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 726..727,
|
||||
value: Int(
|
||||
5,
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
keywords: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
body: [
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 734..737,
|
||||
kind: Shell,
|
||||
value: "ls",
|
||||
},
|
||||
),
|
||||
],
|
||||
orelse: [],
|
||||
},
|
||||
),
|
||||
Assign(
|
||||
StmtAssign {
|
||||
range: 739..748,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 739..741,
|
||||
id: "p1",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
],
|
||||
value: IpyEscapeCommand(
|
||||
ExprIpyEscapeCommand {
|
||||
range: 744..748,
|
||||
kind: Shell,
|
||||
value: "pwd",
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
AnnAssign(
|
||||
StmtAnnAssign {
|
||||
range: 749..763,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 749..751,
|
||||
id: "p2",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
annotation: Name(
|
||||
ExprName {
|
||||
range: 753..756,
|
||||
id: "str",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
value: Some(
|
||||
IpyEscapeCommand(
|
||||
ExprIpyEscapeCommand {
|
||||
range: 759..763,
|
||||
kind: Shell,
|
||||
value: "pwd",
|
||||
},
|
||||
),
|
||||
),
|
||||
simple: true,
|
||||
},
|
||||
),
|
||||
Assign(
|
||||
StmtAssign {
|
||||
range: 764..784,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 764..767,
|
||||
id: "foo",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
],
|
||||
value: IpyEscapeCommand(
|
||||
ExprIpyEscapeCommand {
|
||||
range: 770..784,
|
||||
kind: Magic,
|
||||
value: "foo bar",
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 786..791,
|
||||
kind: Magic,
|
||||
value: " foo",
|
||||
},
|
||||
),
|
||||
Assign(
|
||||
StmtAssign {
|
||||
range: 792..813,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 792..795,
|
||||
id: "foo",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
],
|
||||
value: IpyEscapeCommand(
|
||||
ExprIpyEscapeCommand {
|
||||
range: 798..813,
|
||||
kind: Magic,
|
||||
value: "foo # comment",
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 838..842,
|
||||
kind: Help,
|
||||
value: "foo",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 843..852,
|
||||
kind: Help2,
|
||||
value: "foo.bar",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 853..865,
|
||||
kind: Help,
|
||||
value: "foo.bar.baz",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 866..874,
|
||||
kind: Help2,
|
||||
value: "foo[0]",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 875..885,
|
||||
kind: Help,
|
||||
value: "foo[0][1]",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 886..905,
|
||||
kind: Help2,
|
||||
value: "foo.bar[0].baz[1]",
|
||||
},
|
||||
),
|
||||
IpyEscapeCommand(
|
||||
StmtIpyEscapeCommand {
|
||||
range: 906..929,
|
||||
kind: Help2,
|
||||
value: "foo.bar[0].baz[2].egg",
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/src/parser/tests.rs
|
||||
expression: parse_ast
|
||||
---
|
||||
[
|
||||
Assign(
|
||||
StmtAssign {
|
||||
range: 0..37,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 0..1,
|
||||
id: "x",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
],
|
||||
value: StringLiteral(
|
||||
ExprStringLiteral {
|
||||
range: 4..37,
|
||||
value: StringLiteralValue {
|
||||
inner: Single(
|
||||
StringLiteral {
|
||||
range: 4..37,
|
||||
value: "\u{8}another cool trick",
|
||||
flags: StringLiteralFlags {
|
||||
quote_style: Double,
|
||||
prefix: Empty,
|
||||
triple_quoted: false,
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
]
|
||||
3461
crates/ruff_python_parser/src/parser/statement.rs
Normal file
3461
crates/ruff_python_parser/src/parser/statement.rs
Normal file
File diff suppressed because it is too large
Load diff
152
crates/ruff_python_parser/src/parser/tests.rs
Normal file
152
crates/ruff_python_parser/src/parser/tests.rs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
use crate::{lex, parse, parse_expression, parse_suite, parse_tokens, Mode};
|
||||
|
||||
#[test]
|
||||
fn test_modes() {
|
||||
let source = "a[0][1][2][3][4]";
|
||||
|
||||
assert!(parse(source, Mode::Expression).is_ok());
|
||||
assert!(parse(source, Mode::Module).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expr_mode_invalid_syntax1() {
|
||||
let source = "first second";
|
||||
let error = parse_expression(source).unwrap_err();
|
||||
|
||||
insta::assert_debug_snapshot!(error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expr_mode_invalid_syntax2() {
|
||||
let source = r"first
|
||||
|
||||
second
|
||||
";
|
||||
let error = parse_expression(source).unwrap_err();
|
||||
|
||||
insta::assert_debug_snapshot!(error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expr_mode_invalid_syntax3() {
|
||||
let source = r"first
|
||||
|
||||
second
|
||||
|
||||
third
|
||||
";
|
||||
let error = parse_expression(source).unwrap_err();
|
||||
|
||||
insta::assert_debug_snapshot!(error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expr_mode_valid_syntax() {
|
||||
let source = "first
|
||||
|
||||
";
|
||||
let expr = parse_expression(source).unwrap();
|
||||
|
||||
insta::assert_debug_snapshot!(expr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unicode_aliases() {
|
||||
// https://github.com/RustPython/RustPython/issues/4566
|
||||
let source = r#"x = "\N{BACKSPACE}another cool trick""#;
|
||||
let parse_ast = parse_suite(source).unwrap();
|
||||
|
||||
insta::assert_debug_snapshot!(parse_ast);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ipython_escape_commands() {
|
||||
let parse_ast = parse(
|
||||
r"
|
||||
# Normal Python code
|
||||
(
|
||||
a
|
||||
%
|
||||
b
|
||||
)
|
||||
|
||||
# Dynamic object info
|
||||
??a.foo
|
||||
?a.foo
|
||||
?a.foo?
|
||||
??a.foo()??
|
||||
|
||||
# Line magic
|
||||
%timeit a = b
|
||||
%timeit foo(b) % 3
|
||||
%alias showPath pwd && ls -a
|
||||
%timeit a =\
|
||||
foo(b); b = 2
|
||||
%matplotlib --inline
|
||||
%matplotlib \
|
||||
--inline
|
||||
|
||||
# System shell access
|
||||
!pwd && ls -a | sed 's/^/\ /'
|
||||
!pwd \
|
||||
&& ls -a | sed 's/^/\\ /'
|
||||
!!cd /Users/foo/Library/Application\ Support/
|
||||
|
||||
# Let's add some Python code to make sure that earlier escapes were handled
|
||||
# correctly and that we didn't consume any of the following code as a result
|
||||
# of the escapes.
|
||||
def foo():
|
||||
return (
|
||||
a
|
||||
!=
|
||||
b
|
||||
)
|
||||
|
||||
# Transforms into `foo(..)`
|
||||
/foo 1 2
|
||||
;foo 1 2
|
||||
,foo 1 2
|
||||
|
||||
# Indented escape commands
|
||||
for a in range(5):
|
||||
!ls
|
||||
|
||||
p1 = !pwd
|
||||
p2: str = !pwd
|
||||
foo = %foo \
|
||||
bar
|
||||
|
||||
% foo
|
||||
foo = %foo # comment
|
||||
|
||||
# Help end line magics
|
||||
foo?
|
||||
foo.bar??
|
||||
foo.bar.baz?
|
||||
foo[0]??
|
||||
foo[0][1]?
|
||||
foo.bar[0].baz[1]??
|
||||
foo.bar[0].baz[2].egg??
|
||||
"
|
||||
.trim(),
|
||||
Mode::Ipython,
|
||||
)
|
||||
.unwrap();
|
||||
insta::assert_debug_snapshot!(parse_ast);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ipython_escape_command_parse_error() {
|
||||
let source = r"
|
||||
a = 1
|
||||
%timeit a == 1
|
||||
"
|
||||
.trim();
|
||||
let lxr = lex(source, Mode::Ipython);
|
||||
let parse_err = parse_tokens(lxr.collect(), source, Mode::Module).unwrap_err();
|
||||
assert_eq!(
|
||||
parse_err.to_string(),
|
||||
"IPython escape commands are only allowed in `Mode::Ipython` at byte range 6..20"
|
||||
.to_string()
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue