mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-31 07:37:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Reset FOR_TARGET context for all kinds of parentheses (#11009)

Browse source 
## Summary

This PR fixes a bug in the new parser which involves the parser context
w.r.t. for statement. This is specifically around the `in` keyword which
can be present in the target expression and shouldn't be considered to
be part of the `for` statement header. Ideally it should use a context
which is passed between functions, thus using a call stack to set /
unset a specific variant which will be done in a follow-up PR as it
requires some amount of refactor.

## Test Plan

Add test cases and update the snapshots.
This commit is contained in:
Dhruv Manilawala 2024-04-18 19:37:50 +05:30 • committed by GitHub
parent 13ffb5bc19
commit 8020d486f6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 445 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py Normal file
View file

@ -0,0 +1 @@
for d(x in y) in target: ...

3
crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py
View file

@ -1,2 +1,5 @@
for (x in y)() in iter: ... for (x in y)() in iter: ...
for (x in y) in iter: ... for (x in y) in iter: ...
for (x in y, z) in iter: ...
for [x in y, z] in iter: ...
for {x in y, z} in iter: ...

1
crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py Normal file
View file

@ -0,0 +1 @@
for d[x in y] in target: ...

73
crates/ruff_python_parser/src/parser/expression.rs
View file

@ -360,7 +360,7 @@ impl<'src> Parser<'src> {
} }
// Don't parse a `CompareExpr` if we are parsing a `Comprehension` or `ForStmt` // Don't parse a `CompareExpr` if we are parsing a `Comprehension` or `ForStmt`
if token.is_compare_operator() && self.has_ctx(ParserCtxFlags::FOR_TARGET) { if matches!(token, TokenKind::In) && self.has_ctx(ParserCtxFlags::FOR_TARGET) {
break; break;
} }
@ -659,11 +659,38 @@ impl<'src> Parser<'src> {
Expr::IpyEscapeCommand(self.parse_ipython_escape_command_expression()) Expr::IpyEscapeCommand(self.parse_ipython_escape_command_expression())
} }
TokenKind::String | TokenKind::FStringStart => self.parse_strings(), TokenKind::String | TokenKind::FStringStart => self.parse_strings(),
TokenKind::Lpar => { tok @ (TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace) => {
return self.parse_parenthesized_expression(); // We need to unset the `FOR_TARGET` in the context when parsing an expression
// inside a parentheses, curly brace or brackets otherwise the `in` operator of a
// comparison expression will not be parsed in a `for` target.
// test_ok parenthesized_compare_expr_in_for
// for (x in y)[0] in iter: ...
// for (x in y).attr in iter: ...
// test_err parenthesized_compare_expr_in_for
// for (x in y)() in iter: ...
// for (x in y) in iter: ...
// for (x in y, z) in iter: ...
// for [x in y, z] in iter: ...
// for {x in y, z} in iter: ...
let current_context = self.ctx - ParserCtxFlags::FOR_TARGET;
let saved_context = self.set_ctx(current_context);
let expr = match tok {
TokenKind::Lpar => {
let parsed_expr = self.parse_parenthesized_expression();
self.restore_ctx(current_context, saved_context);
return parsed_expr;
}
TokenKind::Lsqb => self.parse_list_like_expression(),
TokenKind::Lbrace => self.parse_set_or_dict_like_expression(),
_ => unreachable!(),
};
self.restore_ctx(current_context, saved_context);
expr
} }
TokenKind::Lsqb => self.parse_list_like_expression(),
TokenKind::Lbrace => self.parse_set_or_dict_like_expression(),
kind => { kind => {
if kind.is_keyword() { if kind.is_keyword() {
@ -692,14 +719,25 @@ impl<'src> Parser<'src> {
/// ///
/// This method does nothing if the current token is not a candidate for a postfix expression. /// This method does nothing if the current token is not a candidate for a postfix expression.
pub(super) fn parse_postfix_expression(&mut self, mut lhs: Expr, start: TextSize) -> Expr { pub(super) fn parse_postfix_expression(&mut self, mut lhs: Expr, start: TextSize) -> Expr {
loop { // test_ok for_in_target_postfix_expr
// for d[x in y] in target: ...
// test_err for_in_target_postfix_expr
// for d(x in y) in target: ...
let current_context = self.ctx - ParserCtxFlags::FOR_TARGET;
let saved_context = self.set_ctx(current_context);
lhs = loop {
lhs = match self.current_token_kind() { lhs = match self.current_token_kind() {
TokenKind::Lpar => Expr::Call(self.parse_call_expression(lhs, start)), TokenKind::Lpar => Expr::Call(self.parse_call_expression(lhs, start)),
TokenKind::Lsqb => Expr::Subscript(self.parse_subscript_expression(lhs, start)), TokenKind::Lsqb => Expr::Subscript(self.parse_subscript_expression(lhs, start)),
TokenKind::Dot => Expr::Attribute(self.parse_attribute_expression(lhs, start)), TokenKind::Dot => Expr::Attribute(self.parse_attribute_expression(lhs, start)),
_ => break lhs, _ => break lhs,
}; };
} };
self.restore_ctx(current_context, saved_context);
lhs
} }
/// Parse a call expression. /// Parse a call expression.
@ -1748,26 +1786,13 @@ impl<'src> Parser<'src> {
.into(); .into();
} }
// We need to unset the `FOR_TARGET` in the context when parsing a parenthesized expression
// otherwise a parenthesized comparison expression will not be parsed in a `for` target.
// test_ok parenthesized_compare_expr_in_for
// for (x in y)[0] in iter: ...
// for (x in y).attr in iter: ...
// test_err parenthesized_compare_expr_in_for
// for (x in y)() in iter: ...
// for (x in y) in iter: ...
let current_context = self.ctx - ParserCtxFlags::FOR_TARGET;
let saved_context = self.set_ctx(current_context);
// Use the more general rule of the three to parse the first element // Use the more general rule of the three to parse the first element
// and limit it later. // and limit it later.
let mut parsed_expr = self.parse_yield_expression_or_else(|p| { let mut parsed_expr = self.parse_yield_expression_or_else(|p| {
p.parse_star_expression_or_higher(AllowNamedExpression::Yes) p.parse_star_expression_or_higher(AllowNamedExpression::Yes)
}); });
let parsed_expr = match self.current_token_kind() { match self.current_token_kind() {
TokenKind::Comma => { TokenKind::Comma => {
// grammar: `tuple` // grammar: `tuple`
let tuple = self.parse_tuple_expression( let tuple = self.parse_tuple_expression(
@ -1812,11 +1837,7 @@ impl<'src> Parser<'src> {
parsed_expr.is_parenthesized = true; parsed_expr.is_parenthesized = true;
parsed_expr parsed_expr
} }
}; }
self.restore_ctx(current_context, saved_context);
parsed_expr
} }
/// Parses multiple items separated by a comma into a tuple expression. /// Parses multiple items separated by a comma into a tuple expression.

89
crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_in_target_postfix_expr.py.snap Normal file
View file

@ -0,0 +1,89 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py
---
## AST
```
Module(
ModModule {
range: 0..29,
body: [
For(
StmtFor {
range: 0..28,
is_async: false,
target: Call(
ExprCall {
range: 4..13,
func: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
arguments: Arguments {
range: 5..13,
args: [
Compare(
ExprCompare {
range: 6..12,
left: Name(
ExprName {
range: 6..7,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 11..12,
id: "y",
ctx: Load,
},
),
],
},
),
],
keywords: [],
},
},
),
iter: Name(
ExprName {
range: 17..23,
id: "target",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 25..28,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 25..28,
},
),
},
),
],
orelse: [],
},
),
],
},
)
```
## Errors
|
1 | for d(x in y) in target: ...
| ^^^^^^^^^ Syntax Error: Invalid assignment target
|

227
crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_compare_expr_in_for.py.snap
View file

@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/inline/err/parenthesized_compare
``` ```
Module( Module(
ModModule { ModModule {
range: 0..54, range: 0..141,
body: [ body: [
For( For(
StmtFor { StmtFor {
@ -119,6 +119,201 @@ Module(
orelse: [], orelse: [],
}, },
), ),
For(
StmtFor {
range: 54..82,
is_async: false,
target: Tuple(
ExprTuple {
range: 58..69,
elts: [
Compare(
ExprCompare {
range: 59..65,
left: Name(
ExprName {
range: 59..60,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 64..65,
id: "y",
ctx: Load,
},
),
],
},
),
Name(
ExprName {
range: 67..68,
id: "z",
ctx: Store,
},
),
],
ctx: Store,
parenthesized: true,
},
),
iter: Name(
ExprName {
range: 73..77,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 79..82,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 79..82,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 83..111,
is_async: false,
target: List(
ExprList {
range: 87..98,
elts: [
Compare(
ExprCompare {
range: 88..94,
left: Name(
ExprName {
range: 88..89,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 93..94,
id: "y",
ctx: Load,
},
),
],
},
),
Name(
ExprName {
range: 96..97,
id: "z",
ctx: Store,
},
),
],
ctx: Store,
},
),
iter: Name(
ExprName {
range: 102..106,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 108..111,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 108..111,
},
),
},
),
],
orelse: [],
},
),
For(
StmtFor {
range: 112..140,
is_async: false,
target: Set(
ExprSet {
range: 116..127,
elts: [
Compare(
ExprCompare {
range: 117..123,
left: Name(
ExprName {
range: 117..118,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 122..123,
id: "y",
ctx: Load,
},
),
],
},
),
Name(
ExprName {
range: 125..126,
id: "z",
ctx: Load,
},
),
],
},
),
iter: Name(
ExprName {
range: 131..135,
id: "iter",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 137..140,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 137..140,
},
),
},
),
],
orelse: [],
},
),
], ],
}, },
) )
@ -129,6 +324,7 @@ Module(
1 | for (x in y)() in iter: ... 1 | for (x in y)() in iter: ...
| ^^^^^^^^^^ Syntax Error: Invalid assignment target | ^^^^^^^^^^ Syntax Error: Invalid assignment target
2 | for (x in y) in iter: ... 2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
| |
@ -136,4 +332,33 @@ Module(
1 | for (x in y)() in iter: ... 1 | for (x in y)() in iter: ...
2 | for (x in y) in iter: ... 2 | for (x in y) in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target | ^^^^^^ Syntax Error: Invalid assignment target
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
|
|
1 | for (x in y)() in iter: ...
2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target
4 | for [x in y, z] in iter: ...
5 | for {x in y, z} in iter: ...
|
|
2 | for (x in y) in iter: ...
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
| ^^^^^^ Syntax Error: Invalid assignment target
5 | for {x in y, z} in iter: ...
|
|
3 | for (x in y, z) in iter: ...
4 | for [x in y, z] in iter: ...
5 | for {x in y, z} in iter: ...
| ^^^^^^^^^^^ Syntax Error: Invalid assignment target
| |

78
crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_postfix_expr.py.snap Normal file
View file

@ -0,0 +1,78 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py
---
## AST
```
Module(
ModModule {
range: 0..29,
body: [
For(
StmtFor {
range: 0..28,
is_async: false,
target: Subscript(
ExprSubscript {
range: 4..13,
value: Name(
ExprName {
range: 4..5,
id: "d",
ctx: Load,
},
),
slice: Compare(
ExprCompare {
range: 6..12,
left: Name(
ExprName {
range: 6..7,
id: "x",
ctx: Load,
},
),
ops: [
In,
],
comparators: [
Name(
ExprName {
range: 11..12,
id: "y",
ctx: Load,
},
),
],
},
),
ctx: Store,
},
),
iter: Name(
ExprName {
range: 17..23,
id: "target",
ctx: Load,
},
),
body: [
Expr(
StmtExpr {
range: 25..28,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 25..28,
},
),
},
),
],
orelse: [],
},
),
],
},
)
```