mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-12 07:35:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add ExpressionContext for expression parsing (#11055)

Browse source 
## Summary

This PR adds a new `ExpressionContext` struct which is used in
expression parsing.

This solves the following problem:
1. Allowing starred expression with different precedence
2. Allowing yield expression in certain context
3. Remove ambiguity with `in` keyword when parsing a `for ... in`
statement

For context, (1) was solved by adding `parse_star_expression_list` and
`parse_star_expression_or_higher` in #10623, (2) was solved by by adding
`parse_yield_expression_or_else` in #10809, and (3) was fixed in #11009.
All of the mentioned functions have been removed in favor of the context
flags.

As mentioned in #11009, an ideal solution would be to implement an
expression context which is what this PR implements. This is passed
around as function parameter and the call stack is used to automatically
reset the context.

### Recovery

How should the parser recover if the target expression is invalid when
an expression can consume the `in` keyword?

1. Should the `in` keyword be part of the target expression?
2. Or, should the expression parsing stop as soon as `in` keyword is
encountered, no matter the expression?

For example:
```python
for yield x in y: ...

# Here, should this be parsed as
for (yield x) in (y): ...
# Or
for (yield x in y): ...
# where the `in iter` part is missing
```

Or, for binary expression parsing:
```python
for x or y in z: ...

# Should this be parsed as
for (x or y) in z: ...
# Or
for (x or y in z): ...
# where the `in iter` part is missing
```

This need not be solved now, but is very easy to change. For context
this PR does the following:
* For binary, comparison, and unary expressions, stop at `in`
* For lambda, yield expressions, consume the `in`

## Test Plan

1. Add test cases for the `for ... in` statement and verify the
snapshots
2. Make sure the existing test suite pass
3. Run the fuzzer for around 3000 generated source code
4. Run the updated logic on a dozen or so open source repositories
(codename "parser-checkouts")
This commit is contained in:
Dhruv Manilawala 2024-04-23 09:49:05 +05:30 • committed by GitHub
parent 62478c3070
commit c30735d4a7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 1151 additions and 869 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/ruff_python_parser/Cargo.toml
View file

@ -18,7 +18,6 @@ ruff_text_size = { path = "../ruff_text_size" }
anyhow = { workspace = true }
bitflags = { workspace = true }
drop_bomb = { workspace = true }
bstr = { workspace = true }
is-macro = { workspace = true }
itertools = { workspace = true }

1
crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py
View file

@ -1 +0,0 @@
for d(x in y) in target: ...

1
crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py
View file

@ -3,4 +3,5 @@ for "a" in x: ...
for *x and y in z: ...
for *x | y in z: ...
for await x in z: ...
for yield x in y: ...
for [x, 1, y, *["a"]] in z: ...

6
crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py Normal file
View file

@ -0,0 +1,6 @@
for x not in y in z: ...
for x == y in z: ...
for x or y in z: ...
for -x in y: ...
for not x in y: ...
for x | y in z: ...

1
crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py → crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py
View file

@ -1,3 +1,4 @@
for d(x in y) in target: ...
for (x in y)() in iter: ...
for (x in y) in iter: ...
for (x in y, z) in iter: ...

1
crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py
View file

@ -1 +0,0 @@
for d[x in y] in target: ...

1
crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py → crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py
View file

@ -1,2 +1,3 @@
for d[x in y] in target: ...
for (x in y)[0] in iter: ...
for (x in y).attr in iter: ...

524
crates/ruff_python_parser/src/parser/expression.rs
View file

@ -2,6 +2,7 @@ use std::cmp::Ordering;
use std::hash::BuildHasherDefault;
use std::ops::Deref;
use bitflags::bitflags;
use rustc_hash::FxHashSet;
use ruff_python_ast::{
@ -12,7 +13,7 @@ use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use crate::parser::helpers::token_kind_to_cmp_op;
use crate::parser::progress::ParserProgress;
use crate::parser::{helpers, FunctionKind, Parser, ParserCtxFlags};
use crate::parser::{helpers, FunctionKind, Parser};
use crate::string::{parse_fstring_literal_element, parse_string_literal, StringType};
use crate::token_set::TokenSet;
use crate::{FStringErrorType, Mode, ParseErrorType, Tok, TokenKind};
@ -118,29 +119,20 @@ impl<'src> Parser<'src> {
/// Parses every Python expression.
///
/// Matches the `expressions` rule in the [Python grammar].
///
/// The caller can specify whether starred expression is allowed or not. This
/// doesn't affect the parsing of a starred expression as it will be parsed
/// nevertheless. But, if it is not allowed, an error is reported.
///
/// Use [`Parser::parse_star_expression_list`] if the starred expression is
/// required with a bitwise OR precedence.
/// Matches the `expressions` rule in the [Python grammar]. The [`ExpressionContext`] can be
/// used to match the `star_expressions` rule.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
pub(super) fn parse_expression_list(
&mut self,
allow_starred_expression: AllowStarredExpression,
) -> ParsedExpr {
pub(super) fn parse_expression_list(&mut self, context: ExpressionContext) -> ParsedExpr {
let start = self.node_start();
let parsed_expr = self.parse_conditional_expression_or_higher(allow_starred_expression);
let parsed_expr = self.parse_conditional_expression_or_higher_impl(context);
if self.at(TokenKind::Comma) {
Expr::Tuple(self.parse_tuple_expression(
parsed_expr.expr,
start,
Parenthesized::No,
|p| p.parse_conditional_expression_or_higher(allow_starred_expression),
|p| p.parse_conditional_expression_or_higher_impl(context),
))
.into()
} else {
@ -148,76 +140,21 @@ impl<'src> Parser<'src> {
}
}
/// Parses every Python expression.
///
/// Matches the `star_expressions` rule in the [Python grammar].
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
pub(super) fn parse_star_expression_list(&mut self) -> ParsedExpr {
let start = self.node_start();
let parsed_expr = self.parse_star_expression_or_higher(AllowNamedExpression::No);
if self.at(TokenKind::Comma) {
Expr::Tuple(self.parse_tuple_expression(
parsed_expr.expr,
start,
Parenthesized::No,
|parser| parser.parse_star_expression_or_higher(AllowNamedExpression::No),
))
.into()
} else {
parsed_expr
}
}
/// Parses a star expression or any other expression.
///
/// Matches either the `star_named_expression` or `star_expression` rule in
/// the [Python grammar] depending on whether named expressions are allowed
/// or not respectively.
///
/// NOTE: If you have expressions separated by commas and want to parse them
/// individually instead of as a tuple, as done by [`Parser::parse_star_expression_list`],
/// use this function.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
pub(super) fn parse_star_expression_or_higher(
&mut self,
allow_named_expression: AllowNamedExpression,
) -> ParsedExpr {
// This method parses starred expression with a different precedence,
// so don't allow starred expression in other branches.
if self.at(TokenKind::Star) {
Expr::Starred(self.parse_starred_expression(StarredExpressionPrecedence::BitOr)).into()
} else if allow_named_expression.is_yes() {
self.parse_named_expression_or_higher(AllowStarredExpression::No)
} else {
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
}
}
/// Parses every Python expression except unparenthesized tuple.
///
/// Matches the `named_expression` rule in the [Python grammar].
/// Matches the `named_expression` rule in the [Python grammar]. The [`ExpressionContext`] can
/// be used to match the `star_named_expression` rule.
///
/// The caller can specify whether starred expression is allowed or not. This
/// doesn't affect the parsing of a starred expression as it will be parsed
/// nevertheless. But, if it is not allowed, an error is reported.
///
/// Use [`Parser::parse_star_expression_or_higher`] with [`AllowNamedExpression::Yes`]
/// if the starred expression is required with a bitwise OR precedence.
///
/// NOTE: If you have expressions separated by commas and want to parse them
/// individually instead of as a tuple, as done by [`Parser::parse_expression_list`]
/// use this function!
/// NOTE: If you have expressions separated by commas and want to parse them individually
/// instead of as a tuple, as done by [`Parser::parse_expression_list`], use this function.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
pub(super) fn parse_named_expression_or_higher(
&mut self,
allow_starred_expression: AllowStarredExpression,
context: ExpressionContext,
) -> ParsedExpr {
let start = self.node_start();
let parsed_expr = self.parse_conditional_expression_or_higher(allow_starred_expression);
let parsed_expr = self.parse_conditional_expression_or_higher_impl(context);
if self.at(TokenKind::ColonEqual) {
Expr::Named(self.parse_named_expression(parsed_expr.expr, start)).into()
@ -230,27 +167,27 @@ impl<'src> Parser<'src> {
///
/// Matches the `expression` rule in the [Python grammar].
///
/// The caller can specify whether starred expression is allowed or not. This
/// doesn't affect the parsing of a starred expression as it will be parsed
/// nevertheless. But, if it is not allowed, an error is reported.
/// This uses the default [`ExpressionContext`]. Use
/// [`Parser::parse_conditional_expression_or_higher_impl`] if you prefer to pass in the
/// context.
///
/// Use [`Parser::parse_star_expression_or_higher`] with [`AllowNamedExpression::No`]
/// if the starred expression is required with a bitwise OR precedence.
///
/// NOTE: If you have expressions separated by commas and want to parse them
/// individually instead of as a tuple, as done by [`Parser::parse_expression_list`]
/// use this function!
/// NOTE: If you have expressions separated by commas and want to parse them individually
/// instead of as a tuple, as done by [`Parser::parse_expression_list`] use this function.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
pub(super) fn parse_conditional_expression_or_higher(
pub(super) fn parse_conditional_expression_or_higher(&mut self) -> ParsedExpr {
self.parse_conditional_expression_or_higher_impl(ExpressionContext::default())
}
pub(super) fn parse_conditional_expression_or_higher_impl(
&mut self,
allow_starred_expression: AllowStarredExpression,
context: ExpressionContext,
) -> ParsedExpr {
if self.at(TokenKind::Lambda) {
Expr::Lambda(self.parse_lambda_expr()).into()
} else {
let start = self.node_start();
let parsed_expr = self.parse_simple_expression(allow_starred_expression);
let parsed_expr = self.parse_simple_expression(context);
if self.at(TokenKind::If) {
Expr::If(self.parse_if_expression(parsed_expr.expr, start)).into()
@ -266,28 +203,14 @@ impl<'src> Parser<'src> {
/// This is a combination of the `disjunction`, `starred_expression`, `yield_expr`
/// and `lambdef` rules of the [Python grammar].
///
/// Note that this function parses yield and lambda expression but reports an error
/// as they're not allowed in this context. This is done for better error recovery.
/// Use [`Parser::parse_yield_expression_or_else`] to allow parsing yield expression.
/// Use [`Parser::parse_conditional_expression_or_higher`] or any methods which calls
/// into the specified method to allow parsing lambda expression.
///
/// The caller can specify whether starred expression is allowed or not. This
/// doesn't affect the parsing of a starred expression as it will be parsed
/// nevertheless. But, if it is not allowed, an error is reported.
/// Note that this function parses lambda expression but reports an error as they're not
/// allowed in this context. This is done for better error recovery.
/// Use [`Parser::parse_conditional_expression_or_higher`] or any methods which calls into the
/// specified method to allow parsing lambda expression.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
fn parse_simple_expression(
&mut self,
allow_starred_expression: AllowStarredExpression,
) -> ParsedExpr {
let parsed_expr = self.parse_expression_with_precedence(Precedence::Initial);
if allow_starred_expression.is_no() && parsed_expr.is_unparenthesized_starred_expr() {
self.add_error(ParseErrorType::InvalidStarredExpressionUsage, &parsed_expr);
}
parsed_expr
fn parse_simple_expression(&mut self, context: ExpressionContext) -> ParsedExpr {
self.parse_expression_with_precedence(Precedence::Initial, context)
}
/// Returns the binding power of the current token for a Pratt parser.
@ -345,10 +268,14 @@ impl<'src> Parser<'src> {
/// This method uses the [Pratt parsing algorithm].
///
/// [Pratt parsing algorithm]: https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html
fn parse_expression_with_precedence(&mut self, previous_precedence: Precedence) -> ParsedExpr {
fn parse_expression_with_precedence(
&mut self,
previous_precedence: Precedence,
context: ExpressionContext,
) -> ParsedExpr {
let start = self.node_start();
let lhs = self.parse_lhs_expression(previous_precedence);
self.parse_expression_with_precedence_recursive(lhs, previous_precedence, start)
let lhs = self.parse_lhs_expression(previous_precedence, context);
self.parse_expression_with_precedence_recursive(lhs, previous_precedence, context, start)
}
/// Parses an expression with binding power of at least `previous_precedence` given the
@ -357,6 +284,7 @@ impl<'src> Parser<'src> {
&mut self,
mut lhs: ParsedExpr,
previous_precedence: Precedence,
context: ExpressionContext,
start: TextSize,
) -> ParsedExpr {
let mut progress = ParserProgress::default();
@ -369,8 +297,7 @@ impl<'src> Parser<'src> {
break;
}
// Don't parse a `CompareExpr` if we are parsing a `Comprehension` or `ForStmt`
if matches!(token, TokenKind::In) && self.has_ctx(ParserCtxFlags::FOR_TARGET) {
if matches!(token, TokenKind::In) && context.is_in_excluded() {
break;
}
@ -381,29 +308,33 @@ impl<'src> Parser<'src> {
self.bump(token);
// We need to create a dedicated node for boolean operations and
// comparison operations even though they are infix operators.
// We need to create a dedicated node for boolean operations and comparison operations
// even though they are infix operators.
if token.is_bool_operator() {
lhs = Expr::BoolOp(self.parse_bool_operation_expression(
lhs.expr,
start,
token,
operator_binding_power,
))
.into();
continue;
} else if token.is_compare_operator() {
lhs = Expr::Compare(self.parse_compare_expression(
lhs.expr,
start,
token,
operator_binding_power,
context,
))
.into();
continue;
}
let rhs = self.parse_expression_with_precedence(operator_binding_power);
if token.is_compare_operator() {
lhs = Expr::Compare(self.parse_compare_expression(
lhs.expr,
start,
token,
operator_binding_power,
context,
))
.into();
continue;
}
let rhs = self.parse_expression_with_precedence(operator_binding_power, context);
lhs.expr = Expr::BinOp(ast::ExprBinOp {
left: Box::new(lhs.expr),
@ -425,12 +356,16 @@ impl<'src> Parser<'src> {
/// is valid in that context. For example, a unary operator is not valid
/// in an `await` expression in which case the `previous_precedence` would
/// be [`Precedence::Await`].
fn parse_lhs_expression(&mut self, previous_precedence: Precedence) -> ParsedExpr {
fn parse_lhs_expression(
&mut self,
previous_precedence: Precedence,
context: ExpressionContext,
) -> ParsedExpr {
let start = self.node_start();
let lhs = match self.current_token_kind() {
unary_tok @ (TokenKind::Plus | TokenKind::Minus | TokenKind::Tilde) => {
let unary_expr = self.parse_unary_expression();
let unary_expr = self.parse_unary_expression(context);
if previous_precedence > Precedence::PosNegBitNot
// > The power operator `**` binds less tightly than an arithmetic
// > or bitwise unary operator on its right, that is, 2**-1 is 0.5.
@ -448,7 +383,7 @@ impl<'src> Parser<'src> {
Expr::UnaryOp(unary_expr).into()
}
TokenKind::Not => {
let unary_expr = self.parse_unary_expression();
let unary_expr = self.parse_unary_expression(context);
if previous_precedence > Precedence::Not {
self.add_error(
ParseErrorType::OtherError(
@ -460,9 +395,10 @@ impl<'src> Parser<'src> {
Expr::UnaryOp(unary_expr).into()
}
TokenKind::Star => {
let starred_expr =
self.parse_starred_expression(StarredExpressionPrecedence::Conditional);
if previous_precedence > Precedence::Initial {
let starred_expr = self.parse_starred_expression(context);
if previous_precedence > Precedence::Initial
|| !context.is_starred_expression_allowed()
{
self.add_error(ParseErrorType::InvalidStarredExpressionUsage, &starred_expr);
}
Expr::Starred(starred_expr).into()
@ -488,10 +424,12 @@ impl<'src> Parser<'src> {
Expr::Lambda(lambda_expr).into()
}
TokenKind::Yield => {
// Yield expressions aren't allowed in this context but we'll still
// parse it and report an error for better recovery.
let expr = self.parse_yield_expression();
self.add_error(ParseErrorType::InvalidYieldExpressionUsage, &expr);
if previous_precedence > Precedence::Initial
|| !context.is_yield_expression_allowed()
{
self.add_error(ParseErrorType::InvalidYieldExpressionUsage, &expr);
}
expr.into()
}
_ => self.parse_atom(),
@ -511,7 +449,7 @@ impl<'src> Parser<'src> {
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
fn parse_expression_with_bitwise_or_precedence(&mut self) -> ParsedExpr {
let parsed_expr = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let parsed_expr = self.parse_conditional_expression_or_higher();
if parsed_expr.is_parenthesized {
// Parentheses resets the precedence, so we don't need to validate it.
@ -669,38 +607,11 @@ impl<'src> Parser<'src> {
Expr::IpyEscapeCommand(self.parse_ipython_escape_command_expression())
}
TokenKind::String | TokenKind::FStringStart => self.parse_strings(),
tok @ (TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace) => {
// We need to unset the `FOR_TARGET` in the context when parsing an expression
// inside a parentheses, curly brace or brackets otherwise the `in` operator of a
// comparison expression will not be parsed in a `for` target.
// test_ok parenthesized_compare_expr_in_for
// for (x in y)[0] in iter: ...
// for (x in y).attr in iter: ...
// test_err parenthesized_compare_expr_in_for
// for (x in y)() in iter: ...
// for (x in y) in iter: ...
// for (x in y, z) in iter: ...
// for [x in y, z] in iter: ...
// for {x in y, z} in iter: ...
let current_context = self.ctx - ParserCtxFlags::FOR_TARGET;
let saved_context = self.set_ctx(current_context);
let expr = match tok {
TokenKind::Lpar => {
let parsed_expr = self.parse_parenthesized_expression();
self.restore_ctx(current_context, saved_context);
return parsed_expr;
}
TokenKind::Lsqb => self.parse_list_like_expression(),
TokenKind::Lbrace => self.parse_set_or_dict_like_expression(),
_ => unreachable!(),
};
self.restore_ctx(current_context, saved_context);
expr
TokenKind::Lpar => {
return self.parse_parenthesized_expression();
}
TokenKind::Lsqb => self.parse_list_like_expression(),
TokenKind::Lbrace => self.parse_set_or_dict_like_expression(),
kind => {
if kind.is_keyword() {
@ -729,25 +640,14 @@ impl<'src> Parser<'src> {
///
/// This method does nothing if the current token is not a candidate for a postfix expression.
pub(super) fn parse_postfix_expression(&mut self, mut lhs: Expr, start: TextSize) -> Expr {
// test_ok for_in_target_postfix_expr
// for d[x in y] in target: ...
// test_err for_in_target_postfix_expr
// for d(x in y) in target: ...
let current_context = self.ctx - ParserCtxFlags::FOR_TARGET;
let saved_context = self.set_ctx(current_context);
lhs = loop {
loop {
lhs = match self.current_token_kind() {
TokenKind::Lpar => Expr::Call(self.parse_call_expression(lhs, start)),
TokenKind::Lsqb => Expr::Subscript(self.parse_subscript_expression(lhs, start)),
TokenKind::Dot => Expr::Attribute(self.parse_attribute_expression(lhs, start)),
_ => break lhs,
};
};
self.restore_ctx(current_context, saved_context);
lhs
}
}
/// Parse a call expression.
@ -789,8 +689,7 @@ impl<'src> Parser<'src> {
self.parse_comma_separated_list(RecoveryContextKind::Arguments, |parser| {
let argument_start = parser.node_start();
if parser.eat(TokenKind::DoubleStar) {
let value =
parser.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let value = parser.parse_conditional_expression_or_higher();
keywords.push(ast::Keyword {
arg: None,
@ -801,8 +700,8 @@ impl<'src> Parser<'src> {
seen_keyword_unpacking = true;
} else {
let start = parser.node_start();
let mut parsed_expr =
parser.parse_named_expression_or_higher(AllowStarredExpression::Yes);
let mut parsed_expr = parser
.parse_named_expression_or_higher(ExpressionContext::starred_conditional());
match parser.current_token_kind() {
TokenKind::Async | TokenKind::For => {
@ -850,8 +749,7 @@ impl<'src> Parser<'src> {
}
};
let value =
parser.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let value = parser.parse_conditional_expression_or_higher();
keywords.push(ast::Keyword {
arg: Some(arg),
@ -980,7 +878,8 @@ impl<'src> Parser<'src> {
let start = self.node_start();
let lower = if self.at_expr() {
let lower = self.parse_named_expression_or_higher(AllowStarredExpression::Yes);
let lower =
self.parse_named_expression_or_higher(ExpressionContext::starred_conditional());
if self.at_ts(NEWLINE_EOF_SET.union([TokenKind::Rsqb, TokenKind::Comma].into())) {
return lower.expr;
}
@ -1008,20 +907,14 @@ impl<'src> Parser<'src> {
let upper = if self.at_ts(UPPER_END_SET) {
None
} else {
Some(Box::new(
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
))
Some(Box::new(self.parse_conditional_expression_or_higher().expr))
};
let step = if self.eat(TokenKind::Colon) {
if self.at_ts(STEP_END_SET) {
None
} else {
Some(Box::new(
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
))
Some(Box::new(self.parse_conditional_expression_or_higher().expr))
}
} else {
None
@ -1045,7 +938,10 @@ impl<'src> Parser<'src> {
/// If the parser isn't positioned at any of the unary operators.
///
/// See: <https://docs.python.org/3/reference/expressions.html#unary-arithmetic-and-bitwise-operations>
pub(super) fn parse_unary_expression(&mut self) -> ast::ExprUnaryOp {
pub(super) fn parse_unary_expression(
&mut self,
context: ExpressionContext,
) -> ast::ExprUnaryOp {
let start = self.node_start();
let op = UnaryOp::try_from(self.current_token_kind())
@ -1053,10 +949,10 @@ impl<'src> Parser<'src> {
self.bump(self.current_token_kind());
let operand = if op.is_not() {
self.parse_expression_with_precedence(Precedence::Not)
self.parse_expression_with_precedence(Precedence::Not, context)
} else {
// plus, minus and tilde
self.parse_expression_with_precedence(Precedence::PosNegBitNot)
self.parse_expression_with_precedence(Precedence::PosNegBitNot, context)
};
ast::ExprUnaryOp {
@ -1106,6 +1002,7 @@ impl<'src> Parser<'src> {
start: TextSize,
operator_token: TokenKind,
operator_binding_power: Precedence,
context: ExpressionContext,
) -> ast::ExprBoolOp {
let mut values = vec![lhs];
let mut progress = ParserProgress::default();
@ -1115,7 +1012,8 @@ impl<'src> Parser<'src> {
loop {
progress.assert_progressing(self);
let parsed_expr = self.parse_expression_with_precedence(operator_binding_power);
let parsed_expr =
self.parse_expression_with_precedence(operator_binding_power, context);
values.push(parsed_expr.expr);
if !self.eat(operator_token) {
@ -1148,6 +1046,7 @@ impl<'src> Parser<'src> {
start: TextSize,
operator: TokenKind,
operator_binding_power: Precedence,
context: ExpressionContext,
) -> ast::ExprCompare {
let compare_operator = token_kind_to_cmp_op([operator, self.current_token_kind()]).unwrap();
@ -1171,11 +1070,14 @@ impl<'src> Parser<'src> {
loop {
progress.assert_progressing(self);
let parsed_expr = self.parse_expression_with_precedence(operator_binding_power);
let parsed_expr =
self.parse_expression_with_precedence(operator_binding_power, context);
comparators.push(parsed_expr.expr);
let next_operator = self.current_token_kind();
if !next_operator.is_compare_operator() {
if !next_operator.is_compare_operator()
|| (matches!(next_operator, TokenKind::In) && context.is_in_excluded())
{
break;
}
self.bump(next_operator); // compare operator
@ -1525,7 +1427,7 @@ impl<'src> Parser<'src> {
// f"{*}"
// f"{*x and y}"
// f"{*yield x}"
let value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list);
let value = self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
if !value.is_parenthesized && value.expr.is_lambda_expr() {
// TODO(dhruvmanila): This requires making some changes in lambda expression
@ -1655,7 +1557,8 @@ impl<'src> Parser<'src> {
}
// Parse the first element with a more general rule and limit it later.
let first_element = self.parse_star_expression_or_higher(AllowNamedExpression::Yes);
let first_element =
self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or());
match self.current_token_kind() {
TokenKind::Async | TokenKind::For => {
@ -1716,7 +1619,8 @@ impl<'src> Parser<'src> {
// For dictionary expressions, the key uses the `expression` rule while for
// set expressions, the element uses the `star_expression` rule. So, use the
// one that is more general and limit it later.
let key_or_element = self.parse_star_expression_or_higher(AllowNamedExpression::Yes);
let key_or_element =
self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or());
match self.current_token_kind() {
TokenKind::Async | TokenKind::For => {
@ -1747,7 +1651,7 @@ impl<'src> Parser<'src> {
}
self.bump(TokenKind::Colon);
let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let value = self.parse_conditional_expression_or_higher();
if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) {
Expr::DictComp(self.parse_dictionary_comprehension_expression(
@ -1798,19 +1702,16 @@ impl<'src> Parser<'src> {
// Use the more general rule of the three to parse the first element
// and limit it later.
let mut parsed_expr = self.parse_yield_expression_or_else(|p| {
p.parse_star_expression_or_higher(AllowNamedExpression::Yes)
});
let mut parsed_expr =
self.parse_named_expression_or_higher(ExpressionContext::yield_or_starred_bitwise_or());
match self.current_token_kind() {
TokenKind::Comma => {
// grammar: `tuple`
let tuple = self.parse_tuple_expression(
parsed_expr.expr,
start,
Parenthesized::Yes,
|parser| parser.parse_star_expression_or_higher(AllowNamedExpression::Yes),
);
let tuple =
self.parse_tuple_expression(parsed_expr.expr, start, Parenthesized::Yes, |p| {
p.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
});
ParsedExpr {
expr: tuple.into(),
@ -1898,7 +1799,7 @@ impl<'src> Parser<'src> {
self.parse_comma_separated_list(RecoveryContextKind::ListElements, |parser| {
elts.push(
parser
.parse_star_expression_or_higher(AllowNamedExpression::Yes)
.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
.expr,
);
});
@ -1925,7 +1826,7 @@ impl<'src> Parser<'src> {
self.parse_comma_separated_list(RecoveryContextKind::SetElements, |parser| {
elts.push(
parser
.parse_star_expression_or_higher(AllowNamedExpression::Yes)
.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
.expr,
);
});
@ -1962,18 +1863,10 @@ impl<'src> Parser<'src> {
// which requires limiting the expression.
values.push(parser.parse_expression_with_bitwise_or_precedence().expr);
} else {
keys.push(Some(
parser
.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
));
keys.push(Some(parser.parse_conditional_expression_or_higher().expr));
parser.expect(TokenKind::Colon);
values.push(
parser
.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
);
values.push(parser.parse_conditional_expression_or_higher().expr);
}
});
@ -2027,15 +1920,14 @@ impl<'src> Parser<'src> {
self.bump(TokenKind::For);
};
let saved_context = self.set_ctx(ParserCtxFlags::FOR_TARGET);
let mut target = self.parse_expression_list(AllowStarredExpression::Yes);
self.restore_ctx(ParserCtxFlags::FOR_TARGET, saved_context);
let mut target =
self.parse_expression_list(ExpressionContext::starred_conditional().with_in_excluded());
helpers::set_expr_ctx(&mut target.expr, ExprContext::Store);
self.validate_assignment_target(&target.expr);
self.expect(TokenKind::In);
let iter = self.parse_simple_expression(AllowStarredExpression::No);
let iter = self.parse_simple_expression(ExpressionContext::default());
let mut ifs = vec![];
let mut progress = ParserProgress::default();
@ -2043,7 +1935,7 @@ impl<'src> Parser<'src> {
while self.eat(TokenKind::If) {
progress.assert_progressing(self);
let parsed_expr = self.parse_simple_expression(AllowStarredExpression::No);
let parsed_expr = self.parse_simple_expression(ExpressionContext::default());
ifs.push(parsed_expr.expr);
}
@ -2177,18 +2069,15 @@ impl<'src> Parser<'src> {
/// If the parser isn't positioned at a `*` token.
///
/// [Python grammar]: https://docs.python.org/3/reference/grammar.html
fn parse_starred_expression(
&mut self,
precedence: StarredExpressionPrecedence,
) -> ast::ExprStarred {
fn parse_starred_expression(&mut self, context: ExpressionContext) -> ast::ExprStarred {
let start = self.node_start();
self.bump(TokenKind::Star);
let parsed_expr = match precedence {
let parsed_expr = match context.starred_expression_precedence() {
StarredExpressionPrecedence::Conditional => {
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
self.parse_conditional_expression_or_higher_impl(context)
}
StarredExpressionPrecedence::BitOr => {
StarredExpressionPrecedence::BitwiseOr => {
self.parse_expression_with_bitwise_or_precedence()
}
};
@ -2211,7 +2100,8 @@ impl<'src> Parser<'src> {
let start = self.node_start();
self.bump(TokenKind::Await);
let parsed_expr = self.parse_expression_with_precedence(Precedence::Await);
let parsed_expr =
self.parse_expression_with_precedence(Precedence::Await, ExpressionContext::default());
ast::ExprAwait {
value: Box::new(parsed_expr.expr),
@ -2219,23 +2109,6 @@ impl<'src> Parser<'src> {
}
}
/// Parses a yield expression if the parser is positioned at a `yield` token
/// or calls the given closure to parse an expression.
///
/// This method is used where the grammar allows a `yield` expression or an
/// alternative expression. For example, the grammar for a parenthesized
/// expression is `(yield_expr | named_expression)`.
pub(super) fn parse_yield_expression_or_else<F>(&mut self, f: F) -> ParsedExpr
where
F: Fn(&mut Parser<'src>) -> ParsedExpr,
{
if self.at(TokenKind::Yield) {
self.parse_yield_expression().into()
} else {
f(self)
}
}
/// Parses a `yield` expression.
///
/// # Panics
@ -2251,9 +2124,12 @@ impl<'src> Parser<'src> {
return self.parse_yield_from_expression(start);
}
let value = self
.at_expr()
.then(|| Box::new(self.parse_star_expression_list().expr));
let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
});
Expr::Yield(ast::ExprYield {
value,
@ -2283,7 +2159,9 @@ impl<'src> Parser<'src> {
// If we didn't use the `parse_expression_list` method here, the parser
// would have stopped at the comma. Then, the outer expression would
// have been a tuple expression with two elements: `yield from x` and `y`.
let expr = self.parse_expression_list(AllowStarredExpression::No).expr;
let expr = self
.parse_expression_list(ExpressionContext::default())
.expr;
match &expr {
Expr::Tuple(tuple) if !tuple.parenthesized => {
@ -2317,7 +2195,7 @@ impl<'src> Parser<'src> {
}
helpers::set_expr_ctx(&mut target, ExprContext::Store);
let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let value = self.parse_conditional_expression_or_higher();
ast::ExprNamed {
target: Box::new(target),
@ -2364,7 +2242,7 @@ impl<'src> Parser<'src> {
// test_err lambda_body_with_yield_expr
// lambda x: yield y
// lambda x: yield from y
let body = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let body = self.parse_conditional_expression_or_higher();
ast::ExprLambda {
body: Box::new(body.expr),
@ -2383,11 +2261,11 @@ impl<'src> Parser<'src> {
pub(super) fn parse_if_expression(&mut self, body: Expr, start: TextSize) -> ast::ExprIf {
self.bump(TokenKind::If);
let test = self.parse_simple_expression(AllowStarredExpression::No);
let test = self.parse_simple_expression(ExpressionContext::default());
self.expect(TokenKind::Else);
let orelse = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let orelse = self.parse_conditional_expression_or_higher();
ast::ExprIf {
body: Box::new(body),
@ -2589,32 +2467,116 @@ pub(super) enum GeneratorExpressionInParentheses {
},
}
/// Represents the precedence used for parsing the value part of a starred expression.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StarredExpressionPrecedence {
BitOr,
pub(super) enum StarredExpressionPrecedence {
/// Matches `'*' bitwise_or` which is part of the `star_expression` rule in the
/// [Python grammar](https://docs.python.org/3/reference/grammar.html).
BitwiseOr,
/// Matches `'*' expression` which is part of the `starred_expression` rule in the
/// [Python grammar](https://docs.python.org/3/reference/grammar.html).
Conditional,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum AllowNamedExpression {
Yes,
No,
}
/// Represents the expression parsing context.
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)]
pub(super) struct ExpressionContext(ExpressionContextFlags);
impl AllowNamedExpression {
const fn is_yes(self) -> bool {
matches!(self, AllowNamedExpression::Yes)
bitflags! {
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)]
struct ExpressionContextFlags: u8 {
/// This flag is set when the `in` keyword should be excluded from a comparison expression.
/// It is to avoid ambiguity in `for ... in ...` statements.
const EXCLUDE_IN = 1 << 0;
/// This flag is set when a starred expression should be allowed. This doesn't affect the
/// parsing of a starred expression as it will be parsed nevertheless. But, if it is not
/// allowed, an error is reported.
const ALLOW_STARRED_EXPRESSION = 1 << 1;
/// This flag is set when the value of a starred expression should be limited to bitwise OR
/// precedence. Matches the `* bitwise_or` grammar rule if set.
const STARRED_BITWISE_OR_PRECEDENCE = 1 << 2;
/// This flag is set when a yield expression should be allowed. This doesn't affect the
/// parsing of a yield expression as it will be parsed nevertheless. But, if it is not
/// allowed, an error is reported.
const ALLOW_YIELD_EXPRESSION = 1 << 3;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum AllowStarredExpression {
Yes,
No,
}
impl ExpressionContext {
/// Create a new context allowing starred expression at conditional precedence.
pub(super) fn starred_conditional() -> Self {
ExpressionContext::default()
.with_starred_expression_allowed(StarredExpressionPrecedence::Conditional)
}
impl AllowStarredExpression {
const fn is_no(self) -> bool {
matches!(self, AllowStarredExpression::No)
/// Create a new context allowing starred expression at bitwise OR precedence.
pub(super) fn starred_bitwise_or() -> Self {
ExpressionContext::default()
.with_starred_expression_allowed(StarredExpressionPrecedence::BitwiseOr)
}
/// Create a new context allowing starred expression at bitwise OR precedence or yield
/// expression.
pub(super) fn yield_or_starred_bitwise_or() -> Self {
ExpressionContext::starred_bitwise_or().with_yield_expression_allowed()
}
/// Returns a new [`ExpressionContext`] which allows starred expression with the given
/// precedence.
fn with_starred_expression_allowed(self, precedence: StarredExpressionPrecedence) -> Self {
let mut flags = self.0 | ExpressionContextFlags::ALLOW_STARRED_EXPRESSION;
match precedence {
StarredExpressionPrecedence::BitwiseOr => {
flags |= ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE;
}
StarredExpressionPrecedence::Conditional => {
flags -= ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE;
}
}
ExpressionContext(flags)
}
/// Returns a new [`ExpressionContext`] which allows yield expression.
fn with_yield_expression_allowed(self) -> Self {
ExpressionContext(self.0 | ExpressionContextFlags::ALLOW_YIELD_EXPRESSION)
}
/// Returns a new [`ExpressionContext`] which excludes `in` as part of a comparison expression.
pub(super) fn with_in_excluded(self) -> Self {
ExpressionContext(self.0 | ExpressionContextFlags::EXCLUDE_IN)
}
/// Returns `true` if the `in` keyword should be excluded from a comparison expression.
const fn is_in_excluded(self) -> bool {
self.0.contains(ExpressionContextFlags::EXCLUDE_IN)
}
/// Returns `true` if starred expressions are allowed.
const fn is_starred_expression_allowed(self) -> bool {
self.0
.contains(ExpressionContextFlags::ALLOW_STARRED_EXPRESSION)
}
/// Returns `true` if yield expressions are allowed.
const fn is_yield_expression_allowed(self) -> bool {
self.0
.contains(ExpressionContextFlags::ALLOW_YIELD_EXPRESSION)
}
/// Returns the [`StarredExpressionPrecedence`] for the context, regardless of whether starred
/// expressions are allowed or not.
const fn starred_expression_precedence(self) -> StarredExpressionPrecedence {
if self
.0
.contains(ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE)
{
StarredExpressionPrecedence::BitwiseOr
} else {
StarredExpressionPrecedence::Conditional
}
}
}

52
crates/ruff_python_parser/src/parser/mod.rs
View file

@ -1,7 +1,6 @@
use std::cmp::Ordering;
use bitflags::bitflags;
use drop_bomb::DebugDropBomb;
use ast::Mod;
use ruff_python_ast as ast;
@ -16,7 +15,7 @@ use crate::{
Mode, ParseError, ParseErrorType, Tok, TokenKind,
};
use self::expression::AllowStarredExpression;
use self::expression::ExpressionContext;
mod expression;
mod helpers;
@ -77,13 +76,6 @@ pub(crate) struct Parser<'src> {
/// Stores all the syntax errors found during the parsing.
errors: Vec<ParseError>,
/// This tracks the current expression or statement being parsed.
///
/// The `ctx` is also used to create custom error messages and forbid certain
/// expressions or statements of being parsed. The `ctx` should be empty after
/// an expression or statement is done parsing.
ctx: ParserCtxFlags,
/// Specify the mode in which the code will be parsed.
mode: Mode,
@ -123,7 +115,6 @@ impl<'src> Parser<'src> {
mode,
source,
errors: Vec::new(),
ctx: ParserCtxFlags::empty(),
tokens,
recovery_context: RecoveryContext::empty(),
last_token_end: tokens_range.start(),
@ -136,7 +127,7 @@ impl<'src> Parser<'src> {
pub(crate) fn parse_program(mut self) -> Program {
let ast = if self.mode == Mode::Expression {
let start = self.node_start();
let parsed_expr = self.parse_expression_list(AllowStarredExpression::No);
let parsed_expr = self.parse_expression_list(ExpressionContext::default());
// All of the remaining newlines are actually going to be non-logical newlines.
self.eat(TokenKind::Newline);
@ -185,9 +176,6 @@ impl<'src> Parser<'src> {
}
fn finish(self) -> Vec<ParseError> {
// After parsing, the `ctx` and `ctx_stack` should be empty.
// If it's not, you probably forgot to call `clear_ctx` somewhere.
assert_eq!(self.ctx, ParserCtxFlags::empty());
assert_eq!(
self.current_token_kind(),
TokenKind::EndOfFile,
@ -232,29 +220,6 @@ impl<'src> Parser<'src> {
merged
}
#[inline]
#[must_use]
fn set_ctx(&mut self, ctx: ParserCtxFlags) -> SavedParserContext {
SavedParserContext {
flags: std::mem::replace(&mut self.ctx, ctx),
bomb: DebugDropBomb::new(
"You must restore the old parser context explicit by calling `restore_ctx`",
),
}
}
#[inline]
fn restore_ctx(&mut self, current: ParserCtxFlags, mut saved_context: SavedParserContext) {
assert_eq!(self.ctx, current);
saved_context.bomb.defuse();
self.ctx = saved_context.flags;
}
#[inline]
fn has_ctx(&self, ctx: ParserCtxFlags) -> bool {
self.ctx.intersects(ctx)
}
/// Returns the start position for a node that starts at the current token.
fn node_start(&self) -> TextSize {
self.current_token_range().start()
@ -675,13 +640,6 @@ impl SequenceMatchPatternParentheses {
}
}
bitflags! {
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
struct ParserCtxFlags: u8 {
const FOR_TARGET = 1 << 2;
}
}
#[derive(Debug, PartialEq, Copy, Clone)]
enum FunctionKind {
/// A lambda expression, e.g., `lambda x: x`
@ -1327,9 +1285,3 @@ impl RecoveryContext {
})
}
}
#[derive(Debug)]
struct SavedParserContext {
flags: ParserCtxFlags,
bomb: DebugDropBomb,
}

4
crates/ruff_python_parser/src/parser/pattern.rs
View file

@ -6,6 +6,8 @@ use crate::parser::{recovery, Parser, RecoveryContextKind, SequenceMatchPatternP
use crate::token_set::TokenSet;
use crate::{ParseErrorType, Tok, TokenKind};
use super::expression::ExpressionContext;
/// The set of tokens that can start a literal pattern.
const LITERAL_PATTERN_START_SET: TokenSet = TokenSet::new([
TokenKind::None,
@ -483,7 +485,7 @@ impl<'src> Parser<'src> {
TokenKind::Int | TokenKind::Float | TokenKind::Complex
) =>
{
let unary_expr = self.parse_unary_expression();
let unary_expr = self.parse_unary_expression(ExpressionContext::default());
if unary_expr.op.is_u_add() {
self.add_error(

129
crates/ruff_python_parser/src/parser/statement.rs
View file

@ -11,13 +11,12 @@ use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::parser::expression::{GeneratorExpressionInParentheses, ParsedExpr, EXPR_SET};
use crate::parser::progress::ParserProgress;
use crate::parser::{
helpers, FunctionKind, Parser, ParserCtxFlags, RecoveryContext, RecoveryContextKind,
WithItemKind,
helpers, FunctionKind, Parser, RecoveryContext, RecoveryContextKind, WithItemKind,
};
use crate::token_set::TokenSet;
use crate::{Mode, ParseErrorType, Tok, TokenKind};
use super::expression::{AllowNamedExpression, AllowStarredExpression, Precedence};
use super::expression::{ExpressionContext, Precedence};
use super::Parenthesized;
/// Tokens that represent compound statements.
@ -262,7 +261,7 @@ impl<'src> Parser<'src> {
// simple_stmt: `... | yield_stmt | star_expressions | ...`
let parsed_expr =
self.parse_yield_expression_or_else(Parser::parse_star_expression_list);
self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
if self.at(TokenKind::Equal) {
Stmt::Assign(self.parse_assign_statement(parsed_expr, start))
@ -309,8 +308,9 @@ impl<'src> Parser<'src> {
|parser| {
// Allow starred expression to raise a better error message for
// an invalid delete target later.
let mut target =
parser.parse_conditional_expression_or_higher(AllowStarredExpression::Yes);
let mut target = parser.parse_conditional_expression_or_higher_impl(
ExpressionContext::starred_conditional(),
);
helpers::set_expr_ctx(&mut target.expr, ExprContext::Del);
// test_err invalid_del_target
@ -356,9 +356,12 @@ impl<'src> Parser<'src> {
// return yield from x
// return x := 1
// return *x and y
let value = self
.at_expr()
.then(|| Box::new(self.parse_star_expression_list().expr));
let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
});
ast::StmtReturn {
range: self.node_range(start),
@ -384,7 +387,7 @@ impl<'src> Parser<'src> {
// raise *x
// raise yield x
// raise x := 1
let exc = self.parse_expression_list(AllowStarredExpression::No);
let exc = self.parse_expression_list(ExpressionContext::default());
if let Some(ast::ExprTuple {
parenthesized: false,
@ -406,7 +409,7 @@ impl<'src> Parser<'src> {
// raise x from *y
// raise x from yield y
// raise x from y := 1
let cause = self.parse_expression_list(AllowStarredExpression::No);
let cause = self.parse_expression_list(ExpressionContext::default());
if let Some(ast::ExprTuple {
parenthesized: false,
@ -714,7 +717,7 @@ impl<'src> Parser<'src> {
// assert assert x
// assert yield x
// assert x := 1
let test = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let test = self.parse_conditional_expression_or_higher();
let msg = if self.eat(TokenKind::Comma) {
if self.at_expr() {
@ -723,10 +726,7 @@ impl<'src> Parser<'src> {
// assert False, assert x
// assert False, yield x
// assert False, x := 1
Some(Box::new(
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
))
Some(Box::new(self.parse_conditional_expression_or_higher().expr))
} else {
// test_err assert_empty_msg
// assert x,
@ -854,7 +854,7 @@ impl<'src> Parser<'src> {
// type x = yield y
// type x = yield from y
// type x = x := 1
let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let value = self.parse_conditional_expression_or_higher();
ast::StmtTypeAlias {
name: Box::new(name),
@ -1014,7 +1014,8 @@ impl<'src> Parser<'src> {
// x = *lambda x: x
// x = x := 1
let mut value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list);
let mut value =
self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
if self.at(TokenKind::Equal) {
// This path is only taken when there are more than one assignment targets.
@ -1022,7 +1023,7 @@ impl<'src> Parser<'src> {
parser.bump(TokenKind::Equal);
let mut parsed_expr =
parser.parse_yield_expression_or_else(Parser::parse_star_expression_list);
parser.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
std::mem::swap(&mut value, &mut parsed_expr);
@ -1092,7 +1093,7 @@ impl<'src> Parser<'src> {
// test_err ann_assign_stmt_type_alias_annotation
// a: type X = int
// lambda: type X = int
let annotation = self.parse_conditional_expression_or_higher(AllowStarredExpression::No);
let annotation = self.parse_conditional_expression_or_higher();
let value = if self.eat(TokenKind::Equal) {
if self.at_expr() {
@ -1101,7 +1102,7 @@ impl<'src> Parser<'src> {
// x: Any = x := 1
// x: list = [x, *a | b, *a or b]
Some(Box::new(
self.parse_yield_expression_or_else(Parser::parse_star_expression_list)
self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or())
.expr,
))
} else {
@ -1170,7 +1171,7 @@ impl<'src> Parser<'src> {
// x += *yield from x
// x += *lambda x: x
// x += y := 1
let value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list);
let value = self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
ast::StmtAugAssign {
target: Box::new(target.expr),
@ -1198,7 +1199,7 @@ impl<'src> Parser<'src> {
// test_err if_stmt_missing_test
// if : ...
let test = self.parse_named_expression_or_higher(AllowStarredExpression::No);
let test = self.parse_named_expression_or_higher(ExpressionContext::default());
// test_err if_stmt_missing_colon
// if x
@ -1253,7 +1254,7 @@ impl<'src> Parser<'src> {
// elif yield x:
// pass
Some(
self.parse_named_expression_or_higher(AllowStarredExpression::No)
self.parse_named_expression_or_higher(ExpressionContext::default())
.expr,
)
} else {
@ -1414,7 +1415,7 @@ impl<'src> Parser<'src> {
// pass
// except* *x:
// pass
let parsed_expr = self.parse_expression_list(AllowStarredExpression::No);
let parsed_expr = self.parse_expression_list(ExpressionContext::default());
if matches!(
parsed_expr.expr,
Expr::Tuple(ast::ExprTuple {
@ -1522,22 +1523,31 @@ impl<'src> Parser<'src> {
fn parse_for_statement(&mut self, start: TextSize) -> ast::StmtFor {
self.bump(TokenKind::For);
// This is to avoid the ambiguity of the `in` token which is used in
// both the `for` statement and the comparison expression. For example:
//
// ```python
// for x in y:
// # ^^^^^^
// # This is not a comparison expression
// pass
// ```
let saved_context = self.set_ctx(ParserCtxFlags::FOR_TARGET);
// test_err for_stmt_missing_target
// for in x: ...
let mut target = self.parse_expression_list(AllowStarredExpression::Yes);
self.restore_ctx(ParserCtxFlags::FOR_TARGET, saved_context);
// test_ok for_in_target_valid_expr
// for d[x in y] in target: ...
// for (x in y)[0] in iter: ...
// for (x in y).attr in iter: ...
// test_err for_stmt_invalid_target_in_keyword
// for d(x in y) in target: ...
// for (x in y)() in iter: ...
// for (x in y) in iter: ...
// for (x in y, z) in iter: ...
// for [x in y, z] in iter: ...
// for {x in y, z} in iter: ...
// test_err for_stmt_invalid_target_binary_expr
// for x not in y in z: ...
// for x == y in z: ...
// for x or y in z: ...
// for -x in y: ...
// for not x in y: ...
// for x | y in z: ...
let mut target =
self.parse_expression_list(ExpressionContext::starred_conditional().with_in_excluded());
helpers::set_expr_ctx(&mut target.expr, ExprContext::Store);
@ -1547,6 +1557,7 @@ impl<'src> Parser<'src> {
// for *x and y in z: ...
// for *x | y in z: ...
// for await x in z: ...
// for yield x in y: ...
// for [x, 1, y, *["a"]] in z: ...
self.validate_assignment_target(&target.expr);
@ -1563,7 +1574,7 @@ impl<'src> Parser<'src> {
// for x in *a and b: ...
// for x in yield a: ...
// for target in x := 1: ...
let iter = self.parse_star_expression_list();
let iter = self.parse_expression_list(ExpressionContext::starred_bitwise_or());
self.expect(TokenKind::Colon);
@ -1607,7 +1618,7 @@ impl<'src> Parser<'src> {
// while yield x: ...
// while a, b: ...
// while a := 1, b: ...
let test = self.parse_named_expression_or_higher(AllowStarredExpression::No);
let test = self.parse_named_expression_or_higher(ExpressionContext::default());
// test_err while_stmt_missing_colon
// while (
@ -1689,7 +1700,7 @@ impl<'src> Parser<'src> {
// def foo() -> *int: ...
// def foo() -> (*int): ...
// def foo() -> yield x: ...
let returns = self.parse_expression_list(AllowStarredExpression::No);
let returns = self.parse_expression_list(ExpressionContext::default());
if matches!(
returns.expr,
@ -2165,6 +2176,7 @@ impl<'src> Parser<'src> {
self.parse_expression_with_precedence_recursive(
lhs.into(),
Precedence::Initial,
ExpressionContext::default(),
start,
)
.expr
@ -2215,9 +2227,8 @@ impl<'src> Parser<'src> {
//
// Thus, we can conclude that the grammar used should be:
// (yield_expr | star_named_expression)
let parsed_expr = self.parse_yield_expression_or_else(|p| {
p.parse_star_expression_or_higher(AllowNamedExpression::Yes)
});
let parsed_expr = self
.parse_named_expression_or_higher(ExpressionContext::yield_or_starred_bitwise_or());
if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) {
if parsed_expr.is_unparenthesized_starred_expr() {
@ -2279,7 +2290,7 @@ impl<'src> Parser<'src> {
} else {
// If it's not in an ambiguous state, then the grammar of the with item
// should be used which is `expression`.
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
self.parse_conditional_expression_or_higher()
};
let optional_vars = self
@ -2305,7 +2316,8 @@ impl<'src> Parser<'src> {
fn parse_with_item_optional_vars(&mut self) -> ParsedExpr {
self.bump(TokenKind::As);
let mut target = self.parse_conditional_expression_or_higher(AllowStarredExpression::Yes);
let mut target = self
.parse_conditional_expression_or_higher_impl(ExpressionContext::starred_conditional());
// This has the same semantics as an assignment target.
self.validate_assignment_target(&target.expr);
@ -2336,7 +2348,8 @@ impl<'src> Parser<'src> {
//
// First try with `star_named_expression`, then if there's no comma,
// we'll restrict it to `named_expression`.
let subject = self.parse_star_expression_or_higher(AllowNamedExpression::Yes);
let subject =
self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or());
// test_ok match_stmt_subject_expr
// match x := 1:
@ -2360,7 +2373,7 @@ impl<'src> Parser<'src> {
let subject = if self.at(TokenKind::Comma) {
let tuple =
self.parse_tuple_expression(subject.expr, subject_start, Parenthesized::No, |p| {
p.parse_star_expression_or_higher(AllowNamedExpression::Yes)
p.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
});
Expr::Tuple(tuple).into()
@ -2470,7 +2483,7 @@ impl<'src> Parser<'src> {
// match x:
// case y if yield x: ...
Some(Box::new(
self.parse_named_expression_or_higher(AllowStarredExpression::No)
self.parse_named_expression_or_higher(ExpressionContext::default())
.expr,
))
} else {
@ -2588,7 +2601,7 @@ impl<'src> Parser<'src> {
// @yield x
// @yield from x
// def foo(): ...
let parsed_expr = self.parse_named_expression_or_higher(AllowStarredExpression::No);
let parsed_expr = self.parse_named_expression_or_higher(ExpressionContext::default());
decorators.push(ast::Decorator {
expression: parsed_expr.expr,
@ -2744,7 +2757,9 @@ impl<'src> Parser<'src> {
// def foo(*args: *int or str): ...
// def foo(*args: *yield x): ...
// # def foo(*args: **int): ...
self.parse_star_expression_or_higher(AllowNamedExpression::No)
self.parse_conditional_expression_or_higher_impl(
ExpressionContext::starred_bitwise_or(),
)
}
AllowStarAnnotation::No => {
// test_ok param_with_annotation
@ -2757,7 +2772,7 @@ impl<'src> Parser<'src> {
// def foo(arg: *int): ...
// def foo(arg: yield int): ...
// def foo(arg: x := int): ...
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
self.parse_conditional_expression_or_higher()
}
};
Some(Box::new(parsed_expr.expr))
@ -2809,10 +2824,7 @@ impl<'src> Parser<'src> {
// def foo(x=*int): ...
// def foo(x=(*int)): ...
// def foo(x=yield y): ...
Some(Box::new(
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
))
Some(Box::new(self.parse_conditional_expression_or_higher().expr))
} else {
// test_err param_missing_default
// def foo(x=): ...
@ -3176,10 +3188,7 @@ impl<'src> Parser<'src> {
// type X[T: yield x] = int
// type X[T: yield from x] = int
// type X[T: x := int] = int
Some(Box::new(
self.parse_conditional_expression_or_higher(AllowStarredExpression::No)
.expr,
))
Some(Box::new(self.parse_conditional_expression_or_higher().expr))
} else {
// test_err type_param_missing_bound
// type X[T: ] = int

39
crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap
View file

@ -346,24 +346,29 @@ Module(
ExprList {
range: 187..199,
elts: [
Starred(
ExprStarred {
range: 188..190,
value: Name(
ExprName {
range: 189..190,
id: "x",
ctx: Load,
Named(
ExprNamed {
range: 188..195,
target: Starred(
ExprStarred {
range: 188..190,
value: Name(
ExprName {
range: 189..190,
id: "x",
ctx: Store,
},
),
ctx: Store,
},
),
ctx: Load,
},
),
NumberLiteral(
ExprNumberLiteral {
range: 194..195,
value: Int(
2,
value: NumberLiteral(
ExprNumberLiteral {
range: 194..195,
value: Int(
2,
),
},
),
},
),
@ -458,5 +463,5 @@ Module(
8 | [*x if True else y, z]
9 | [*lambda x: x, z]
10 | [*x := 2, z]
| ^^ Syntax Error: Expected ',', found ':='
| ^^ Syntax Error: Assignment expression target must be an identifier
|

73
crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__named__invalid_target.py.snap
View file

@ -84,30 +84,30 @@ Module(
),
Expr(
StmtExpr {
range: 81..84,
value: Starred(
ExprStarred {
range: 82..84,
value: Name(
ExprName {
range: 83..84,
id: "x",
ctx: Load,
range: 81..90,
value: Named(
ExprNamed {
range: 82..89,
target: Starred(
ExprStarred {
range: 82..84,
value: Name(
ExprName {
range: 83..84,
id: "x",
ctx: Store,
},
),
ctx: Store,
},
),
ctx: Load,
},
),
},
),
Expr(
StmtExpr {
range: 88..89,
value: NumberLiteral(
ExprNumberLiteral {
range: 88..89,
value: Int(
1,
value: NumberLiteral(
ExprNumberLiteral {
range: 88..89,
value: Int(
1,
),
},
),
},
),
@ -198,34 +198,7 @@ Module(
3 | (x.y := 1)
4 | (x[y] := 1)
5 | (*x := 1)
| ^^ Syntax Error: Starred expression cannot be used here
6 | ([x, y] := [1, 2])
|
|
3 | (x.y := 1)
4 | (x[y] := 1)
5 | (*x := 1)
| ^^ Syntax Error: Expected ')', found ':='
6 | ([x, y] := [1, 2])
|
|
3 | (x.y := 1)
4 | (x[y] := 1)
5 | (*x := 1)
| ^ Syntax Error: Expected a statement
6 | ([x, y] := [1, 2])
|
|
3 | (x.y := 1)
4 | (x[y] := 1)
5 | (*x := 1)
| ^ Syntax Error: Expected a statement
| ^^ Syntax Error: Assignment expression target must be an identifier
6 | ([x, y] := [1, 2])
|

120
crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap
View file

@ -491,34 +491,34 @@ Module(
),
Expr(
StmtExpr {
range: 323..326,
value: Starred(
ExprStarred {
range: 324..326,
value: Name(
ExprName {
range: 325..326,
id: "x",
ctx: Load,
},
),
ctx: Load,
},
),
},
),
Expr(
StmtExpr {
range: 330..343,
range: 323..344,
value: Tuple(
ExprTuple {
range: 330..343,
range: 323..344,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 330..331,
value: Int(
2,
Named(
ExprNamed {
range: 324..331,
target: Starred(
ExprStarred {
range: 324..326,
value: Name(
ExprName {
range: 325..326,
id: "x",
ctx: Store,
},
),
ctx: Store,
},
),
value: NumberLiteral(
ExprNumberLiteral {
range: 330..331,
value: Int(
2,
),
},
),
},
),
@ -529,30 +529,35 @@ Module(
ctx: Load,
},
),
Starred(
ExprStarred {
range: 336..338,
value: Name(
ExprName {
range: 337..338,
id: "x",
ctx: Load,
Named(
ExprNamed {
range: 336..343,
target: Starred(
ExprStarred {
range: 336..338,
value: Name(
ExprName {
range: 337..338,
id: "x",
ctx: Store,
},
),
ctx: Store,
},
),
ctx: Load,
},
),
NumberLiteral(
ExprNumberLiteral {
range: 342..343,
value: Int(
2,
value: NumberLiteral(
ExprNumberLiteral {
range: 342..343,
value: Int(
2,
),
},
),
},
),
],
ctx: Load,
parenthesized: false,
parenthesized: true,
},
),
},
@ -1231,7 +1236,7 @@ Module(
8 | (*x if True else y, z, *x if True else y)
9 | (*lambda x: x, z, *lambda x: x)
10 | (*x := 2, z, *x := 2)
| ^^ Syntax Error: Starred expression cannot be used here
| ^^ Syntax Error: Assignment expression target must be an identifier
|
@ -1239,34 +1244,7 @@ Module(
8 | (*x if True else y, z, *x if True else y)
9 | (*lambda x: x, z, *lambda x: x)
10 | (*x := 2, z, *x := 2)
| ^^ Syntax Error: Expected ')', found ':='
|
|
8 | (*x if True else y, z, *x if True else y)
9 | (*lambda x: x, z, *lambda x: x)
10 | (*x := 2, z, *x := 2)
| ^^ Syntax Error: Expected ',', found ':='
|
|
8 | (*x if True else y, z, *x if True else y)
9 | (*lambda x: x, z, *lambda x: x)
10 | (*x := 2, z, *x := 2)
| ^ Syntax Error: Expected a statement
|
|
8 | (*x if True else y, z, *x if True else y)
9 | (*lambda x: x, z, *lambda x: x)
10 | (*x := 2, z, *x := 2)
| ^ Syntax Error: Expected a statement
11 |
12 |
13 | # Non-parenthesized
| ^^ Syntax Error: Assignment expression target must be an identifier
|

39
crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap
View file

@ -339,24 +339,29 @@ Module(
ExprSet {
range: 186..198,
elts: [
Starred(
ExprStarred {
range: 187..189,
value: Name(
ExprName {
range: 188..189,
id: "x",
ctx: Load,
Named(
ExprNamed {
range: 187..194,
target: Starred(
ExprStarred {
range: 187..189,
value: Name(
ExprName {
range: 188..189,
id: "x",
ctx: Store,
},
),
ctx: Store,
},
),
ctx: Load,
},
),
NumberLiteral(
ExprNumberLiteral {
range: 193..194,
value: Int(
2,
value: NumberLiteral(
ExprNumberLiteral {
range: 193..194,
value: Int(
2,
),
},
),
},
),
@ -450,5 +455,5 @@ Module(
8 | {*x if True else y, z}
9 | {*lambda x: x, z}
10 | {*x := 2, z}
| ^^ Syntax Error: Expected ',', found ':='
| ^^ Syntax Error: Assignment expression target must be an identifier
|

89
crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_in_target_postfix_expr.py.snap
View file

@ -1,89 +0,0 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py
---
## AST
```
Module(
ModModule {
range: 0..29,
body: [
For(
StmtFor {
range: 0..28,
is_async: false,
target: Call(
ExprCall {
range: 4..13,
func: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
arguments: Arguments {
range: 5..13,
args: [
Compare(
ExprCompare {
range: 6..12,
left: Name(
ExprName {
range: 6..7,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 11..12,
id: "y",
ctx: Load,
},
),
],
},
),
],
keywords: [],
},
},
),
iter: Name(
ExprName {
range: 17..23,
id: "target",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 25..28,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 25..28,
},
),
},
),
],
orelse: [],
},
),
],
},
)
```
## Errors
|
1 | for d(x in y) in target: ...
| ^^^^^^^^^ Syntax Error: Invalid assignment target
|

112
crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap
View file

@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_targ
```
Module(
ModModule {
range: 0..132,
range: 0..154,
body: [
For(
StmtFor {
@ -233,22 +233,79 @@ Module(
),
For(
StmtFor {
range: 100..131,
range: 100..121,
is_async: false,
target: Yield(
ExprYield {
range: 104..116,
value: Some(
Compare(
ExprCompare {
range: 110..116,
left: Name(
ExprName {
range: 110..111,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 115..116,
id: "y",
ctx: Load,
},
),
],
},
),
),
},
),
iter: Name(
ExprName {
range: 116..116,
id: "",
ctx: Invalid,
},
),
body: [
Expr(
StmtExpr {
range: 118..121,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 118..121,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 122..153,
is_async: false,
target: List(
ExprList {
range: 104..121,
range: 126..143,
elts: [
Name(
ExprName {
range: 105..106,
range: 127..128,
id: "x",
ctx: Store,
},
),
NumberLiteral(
ExprNumberLiteral {
range: 108..109,
range: 130..131,
value: Int(
1,
),
@ -256,25 +313,25 @@ Module(
),
Name(
ExprName {
range: 111..112,
range: 133..134,
id: "y",
ctx: Store,
},
),
Starred(
ExprStarred {
range: 114..120,
range: 136..142,
value: List(
ExprList {
range: 115..120,
range: 137..142,
elts: [
StringLiteral(
ExprStringLiteral {
range: 116..119,
range: 138..141,
value: StringLiteralValue {
inner: Single(
StringLiteral {
range: 116..119,
range: 138..141,
value: "a",
flags: StringLiteralFlags {
quote_style: Double,
@ -299,7 +356,7 @@ Module(
),
iter: Name(
ExprName {
range: 125..126,
range: 147..148,
id: "z",
ctx: Load,
},
@ -307,10 +364,10 @@ Module(
body: [
Expr(
StmtExpr {
range: 128..131,
range: 150..153,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 128..131,
range: 150..153,
},
),
},
@ -358,7 +415,7 @@ Module(
4 | for *x | y in z: ...
| ^^^^^ Syntax Error: Invalid assignment target
5 | for await x in z: ...
6 | for [x, 1, y, *["a"]] in z: ...
6 | for yield x in y: ...
|
@ -367,21 +424,40 @@ Module(
4 | for *x | y in z: ...
5 | for await x in z: ...
| ^^^^^^^ Syntax Error: Invalid assignment target
6 | for [x, 1, y, *["a"]] in z: ...
6 | for yield x in y: ...
7 | for [x, 1, y, *["a"]] in z: ...
|
|
4 | for *x | y in z: ...
5 | for await x in z: ...
6 | for [x, 1, y, *["a"]] in z: ...
6 | for yield x in y: ...
| ^^^^^^^^^^^^ Syntax Error: Yield expression cannot be used here
7 | for [x, 1, y, *["a"]] in z: ...
|
|
4 | for *x | y in z: ...
5 | for await x in z: ...
6 | for yield x in y: ...
| ^ Syntax Error: Expected 'in', found ':'
7 | for [x, 1, y, *["a"]] in z: ...
|
|
5 | for await x in z: ...
6 | for yield x in y: ...
7 | for [x, 1, y, *["a"]] in z: ...
| ^ Syntax Error: Invalid assignment target
|
|
4 | for *x | y in z: ...
5 | for await x in z: ...
6 | for [x, 1, y, *["a"]] in z: ...
6 | for yield x in y: ...
7 | for [x, 1, y, *["a"]] in z: ...
| ^^^ Syntax Error: Invalid assignment target
|

341
crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap Normal file
View file

@ -0,0 +1,341 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py
---
## AST
```
Module(
ModModule {
range: 0..124,
body: [
For(
StmtFor {
range: 0..24,
is_async: false,
target: Compare(
ExprCompare {
range: 4..14,
left: Name(
ExprName {
range: 4..5,
id: "x",
ctx: Load,
},
),
ops: [
NotIn,
],
comparators: [
Name(
ExprName {
range: 13..14,
id: "y",
ctx: Load,
},
),
],
},
),
iter: Name(
ExprName {
range: 18..19,
id: "z",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 21..24,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 21..24,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 25..45,
is_async: false,
target: Compare(
ExprCompare {
range: 29..35,
left: Name(
ExprName {
range: 29..30,
id: "x",
ctx: Load,
},
),
ops: [
Eq,
],
comparators: [
Name(
ExprName {
range: 34..35,
id: "y",
ctx: Load,
},
),
],
},
),
iter: Name(
ExprName {
range: 39..40,
id: "z",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 42..45,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 42..45,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 46..66,
is_async: false,
target: BoolOp(
ExprBoolOp {
range: 50..56,
op: Or,
values: [
Name(
ExprName {
range: 50..51,
id: "x",
ctx: Load,
},
),
Name(
ExprName {
range: 55..56,
id: "y",
ctx: Load,
},
),
],
},
),
iter: Name(
ExprName {
range: 60..61,
id: "z",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 63..66,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 63..66,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 67..83,
is_async: false,
target: UnaryOp(
ExprUnaryOp {
range: 71..73,
op: USub,
operand: Name(
ExprName {
range: 72..73,
id: "x",
ctx: Store,
},
),
},
),
iter: Name(
ExprName {
range: 77..78,
id: "y",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 80..83,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 80..83,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 84..103,
is_async: false,
target: UnaryOp(
ExprUnaryOp {
range: 88..93,
op: Not,
operand: Name(
ExprName {
range: 92..93,
id: "x",
ctx: Store,
},
),
},
),
iter: Name(
ExprName {
range: 97..98,
id: "y",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 100..103,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 100..103,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 104..123,
is_async: false,
target: BinOp(
ExprBinOp {
range: 108..113,
left: Name(
ExprName {
range: 108..109,
id: "x",
ctx: Load,
},
),
op: BitOr,
right: Name(
ExprName {
range: 112..113,
id: "y",
ctx: Load,
},
),
},
),
iter: Name(
ExprName {
range: 117..118,
id: "z",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 120..123,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 120..123,
},
),
},
),
],
orelse: [],
},
),
],
},
)
```
## Errors
|
1 | for x not in y in z: ...
| ^^^^^^^^^^ Syntax Error: Invalid assignment target
2 | for x == y in z: ...
3 | for x or y in z: ...
|
|
1 | for x not in y in z: ...
2 | for x == y in z: ...
| ^^^^^^ Syntax Error: Invalid assignment target
3 | for x or y in z: ...
4 | for -x in y: ...
|
|
1 | for x not in y in z: ...
2 | for x == y in z: ...
3 | for x or y in z: ...
| ^^^^^^ Syntax Error: Invalid assignment target
4 | for -x in y: ...
5 | for not x in y: ...
|
|
2 | for x == y in z: ...
3 | for x or y in z: ...
4 | for -x in y: ...
| ^^ Syntax Error: Invalid assignment target
5 | for not x in y: ...
6 | for x | y in z: ...
|
|
3 | for x or y in z: ...
4 | for -x in y: ...
5 | for not x in y: ...
| ^^^^^ Syntax Error: Invalid assignment target
6 | for x | y in z: ...
|
|
4 | for -x in y: ...
5 | for not x in y: ...
6 | for x | y in z: ...
| ^^^^^ Syntax Error: Invalid assignment target
|

288
crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_compare_expr_in_for.py.snap → crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap
View file

@ -1,27 +1,95 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py
input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py
---
## AST
```
Module(
ModModule {
range: 0..141,
range: 0..170,
body: [
For(
StmtFor {
range: 0..27,
range: 0..28,
is_async: false,
target: Call(
ExprCall {
range: 4..14,
range: 4..13,
func: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
arguments: Arguments {
range: 5..13,
args: [
Compare(
ExprCompare {
range: 6..12,
left: Name(
ExprName {
range: 6..7,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 11..12,
id: "y",
ctx: Load,
},
),
],
},
),
],
keywords: [],
},
},
),
iter: Name(
ExprName {
range: 17..23,
id: "target",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 25..28,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 25..28,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 29..56,
is_async: false,
target: Call(
ExprCall {
range: 33..43,
func: Compare(
ExprCompare {
range: 5..11,
range: 34..40,
left: Name(
ExprName {
range: 5..6,
range: 34..35,
id: "x",
ctx: Load,
},
@ -32,7 +100,7 @@ Module(
comparators: [
Name(
ExprName {
range: 10..11,
range: 39..40,
id: "y",
ctx: Load,
},
@ -41,7 +109,7 @@ Module(
},
),
arguments: Arguments {
range: 12..14,
range: 41..43,
args: [],
keywords: [],
},
@ -49,7 +117,7 @@ Module(
),
iter: Name(
ExprName {
range: 18..22,
range: 47..51,
id: "iter",
ctx: Load,
},
@ -57,10 +125,10 @@ Module(
body: [
Expr(
StmtExpr {
range: 24..27,
range: 53..56,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 24..27,
range: 53..56,
},
),
},
@ -71,14 +139,14 @@ Module(
),
For(
StmtFor {
range: 28..53,
range: 57..82,
is_async: false,
target: Compare(
ExprCompare {
range: 33..39,
range: 62..68,
left: Name(
ExprName {
range: 33..34,
range: 62..63,
id: "x",
ctx: Load,
},
@ -89,7 +157,7 @@ Module(
comparators: [
Name(
ExprName {
range: 38..39,
range: 67..68,
id: "y",
ctx: Load,
},
@ -97,72 +165,6 @@ Module(
],
},
),
iter: Name(
ExprName {
range: 44..48,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 50..53,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 50..53,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 54..82,
is_async: false,
target: Tuple(
ExprTuple {
range: 58..69,
elts: [
Compare(
ExprCompare {
range: 59..65,
left: Name(
ExprName {
range: 59..60,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 64..65,
id: "y",
ctx: Load,
},
),
],
},
),
Name(
ExprName {
range: 67..68,
id: "z",
ctx: Store,
},
),
],
ctx: Store,
parenthesized: true,
},
),
iter: Name(
ExprName {
range: 73..77,
@ -189,8 +191,8 @@ Module(
StmtFor {
range: 83..111,
is_async: false,
target: List(
ExprList {
target: Tuple(
ExprTuple {
range: 87..98,
elts: [
Compare(
@ -226,6 +228,7 @@ Module(
),
],
ctx: Store,
parenthesized: true,
},
),
iter: Name(
@ -254,8 +257,8 @@ Module(
StmtFor {
range: 112..140,
is_async: false,
target: Set(
ExprSet {
target: List(
ExprList {
range: 116..127,
elts: [
Compare(
@ -286,10 +289,11 @@ Module(
ExprName {
range: 125..126,
id: "z",
ctx: Load,
ctx: Store,
},
),
],
ctx: Store,
},
),
iter: Name(
@ -314,6 +318,70 @@ Module(
orelse: [],
},
),
For(
StmtFor {
range: 141..169,
is_async: false,
target: Set(
ExprSet {
range: 145..156,
elts: [
Compare(
ExprCompare {
range: 146..152,
left: Name(
ExprName {
range: 146..147,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 151..152,
id: "y",
ctx: Load,
},
),
],
},
),
Name(
ExprName {
range: 154..155,
id: "z",
ctx: Load,
},
),
],
},
),
iter: Name(
ExprName {
range: 160..164,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 166..169,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 166..169,
},
),
},
),
],
orelse: [],
},
),
],
},
)
@ -321,44 +389,54 @@ Module(
## Errors
|
1 | for (x in y)() in iter: ...
1 | for d(x in y) in target: ...
| ^^^^^^^^^ Syntax Error: Invalid assignment target
2 | for (x in y)() in iter: ...
3 | for (x in y) in iter: ...
|
|
1 | for d(x in y) in target: ...
2 | for (x in y)() in iter: ...
| ^^^^^^^^^^ Syntax Error: Invalid assignment target
2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
3 | for (x in y) in iter: ...
4 | for (x in y, z) in iter: ...
|
|
1 | for (x in y)() in iter: ...
2 | for (x in y) in iter: ...
1 | for d(x in y) in target: ...
2 | for (x in y)() in iter: ...
3 | for (x in y) in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
4 | for (x in y, z) in iter: ...
5 | for [x in y, z] in iter: ...
|
|
1 | for (x in y)() in iter: ...
2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
2 | for (x in y)() in iter: ...
3 | for (x in y) in iter: ...
4 | for (x in y, z) in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target
4 | for [x in y, z] in iter: ...
5 | for {x in y, z} in iter: ...
5 | for [x in y, z] in iter: ...
6 | for {x in y, z} in iter: ...
|
|
2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
3 | for (x in y) in iter: ...
4 | for (x in y, z) in iter: ...
5 | for [x in y, z] in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target
5 | for {x in y, z} in iter: ...
6 | for {x in y, z} in iter: ...
|
|
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
5 | for {x in y, z} in iter: ...
4 | for (x in y, z) in iter: ...
5 | for [x in y, z] in iter: ...
6 | for {x in y, z} in iter: ...
| ^^^^^^^^^^^ Syntax Error: Invalid assignment target
|

78
crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_postfix_expr.py.snap
View file

@ -1,78 +0,0 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py
---
## AST
```
Module(
ModModule {
range: 0..29,
body: [
For(
StmtFor {
range: 0..28,
is_async: false,
target: Subscript(
ExprSubscript {
range: 4..13,
value: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
slice: Compare(
ExprCompare {
range: 6..12,
left: Name(
ExprName {
range: 6..7,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 11..12,
id: "y",
ctx: Load,
},
),
],
},
),
ctx: Store,
},
),
iter: Name(
ExprName {
range: 17..23,
id: "target",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 25..28,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 25..28,
},
),
},
),
],
orelse: [],
},
),
],
},
)
```

119
crates/ruff_python_parser/tests/snapshots/valid_syntax@parenthesized_compare_expr_in_for.py.snap → crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap
View file

@ -1,13 +1,13 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py
input_file: crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py
---
## AST
```
Module(
ModModule {
range: 0..60,
range: 0..89,
body: [
For(
StmtFor {
@ -15,13 +15,20 @@ Module(
is_async: false,
target: Subscript(
ExprSubscript {
range: 4..15,
value: Compare(
range: 4..13,
value: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
slice: Compare(
ExprCompare {
range: 5..11,
range: 6..12,
left: Name(
ExprName {
range: 5..6,
range: 6..7,
id: "x",
ctx: Load,
},
@ -32,7 +39,7 @@ Module(
comparators: [
Name(
ExprName {
range: 10..11,
range: 11..12,
id: "y",
ctx: Load,
},
@ -40,21 +47,13 @@ Module(
],
},
),
slice: NumberLiteral(
ExprNumberLiteral {
range: 13..14,
value: Int(
0,
),
},
),
ctx: Store,
},
),
iter: Name(
ExprName {
range: 19..23,
id: "iter",
range: 17..23,
id: "target",
ctx: Load,
},
),
@ -75,11 +74,11 @@ Module(
),
For(
StmtFor {
range: 29..59,
range: 29..57,
is_async: false,
target: Attribute(
ExprAttribute {
range: 33..46,
target: Subscript(
ExprSubscript {
range: 33..44,
value: Compare(
ExprCompare {
range: 34..40,
@ -104,16 +103,20 @@ Module(
],
},
),
attr: Identifier {
id: "attr",
range: 42..46,
},
slice: NumberLiteral(
ExprNumberLiteral {
range: 42..43,
value: Int(
0,
),
},
),
ctx: Store,
},
),
iter: Name(
ExprName {
range: 50..54,
range: 48..52,
id: "iter",
ctx: Load,
},
@ -121,10 +124,70 @@ Module(
body: [
Expr(
StmtExpr {
range: 56..59,
range: 54..57,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 56..59,
range: 54..57,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 58..88,
is_async: false,
target: Attribute(
ExprAttribute {
range: 62..75,
value: Compare(
ExprCompare {
range: 63..69,
left: Name(
ExprName {
range: 63..64,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 68..69,
id: "y",
ctx: Load,
},
),
],
},
),
attr: Identifier {
id: "attr",
range: 71..75,
},
ctx: Store,
},
),
iter: Name(
ExprName {
range: 79..83,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 85..88,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 85..88,
},
),
},