mirror of
https://github.com/astral-sh/ruff.git
synced 2025-07-24 13:34:40 +00:00
Use speculative parsing for with-items (#11770)
## Summary This PR updates the with-items parsing logic to use speculative parsing instead. ### Existing logic First, let's understand the previous logic: 1. The parser sees `(`, it doesn't know whether it's part of a parenthesized with items or a parenthesized expression 2. Consider it a parenthesized with items and perform a hand-rolled speculative parsing 3. Then, verify the assumption and if it's incorrect convert the parsed with items into an appropriate expression which becomes part of the first with item Here, in (3) there are lots of edge cases which we've to deal with: 1. Trailing comma with a single element should be [converted to the expression as is](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2140-L2153)) 2. Trailing comma with multiple elements should be [converted to a tuple expression](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2155-L2178)) 3. Limit the allowed expression based on whether it's [(1)](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2144-L2152)) or [(2)](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2157-L2171)) 4. [Consider postfix expressions](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2181-L2200)) after (3) 5. [Consider `if` expressions](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2203-L2208)) after (3) 6. [Consider binary expressions](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2210-L2228)) after (3) Consider other cases like * [Single generator expression](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2020-L2035)) * [Expecting a comma](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2122-L2130)) And, this is all possible only if we allow parsing these expressions in the [with item parsing logic](9b2cf569b2/crates/ruff_python_parser/src/parser/statement.rs (L2287-L2334)). ### Speculative parsing With #11457 merged, we can simplify this logic by changing the step (3) from above to just rewind the parser back to the `(` if our assumption (parenthesized with-items) was incorrect and then continue parsing it considering parenthesized expression. This also behaves a lot similar to what a PEG parser does which is to consider the first grammar rule and if it fails consider the second grammar rule and so on. resolves: #11639 ## Test Plan - [x] Verify the updated snapshots - [x] Run the fuzzer on around 3000 valid source code (locally)
This commit is contained in:
Dhruv Manilawala
GitHub
parent
5a5a588a72
commit
6c1fa1d440
9 changed files with 556 additions and 731 deletions
|
|
@ -0,0 +1,3 @@
|
|||
# `)` followed by a newline
|
||||
with (item1, item2)
|
||||
pass
|
||||
|
|
@ -689,7 +689,8 @@ impl<'src> Parser<'src> {
|
|||
|
||||
parsed_expr = Expr::Generator(parser.parse_generator_expression(
|
||||
parsed_expr.expr,
|
||||
GeneratorExpressionInParentheses::No(start),
|
||||
start,
|
||||
Parenthesized::No,
|
||||
))
|
||||
.into();
|
||||
}
|
||||
|
|
@ -1705,7 +1706,8 @@ impl<'src> Parser<'src> {
|
|||
|
||||
let generator = Expr::Generator(self.parse_generator_expression(
|
||||
parsed_expr.expr,
|
||||
GeneratorExpressionInParentheses::Yes(start),
|
||||
start,
|
||||
Parenthesized::Yes,
|
||||
));
|
||||
|
||||
ParsedExpr {
|
||||
|
|
@ -1929,46 +1931,27 @@ impl<'src> Parser<'src> {
|
|||
|
||||
/// Parses a generator expression.
|
||||
///
|
||||
/// The given `in_parentheses` parameter is used to determine whether the generator
|
||||
/// expression is enclosed in parentheses or not:
|
||||
/// - `Yes`, expect the `)` token after the generator expression.
|
||||
/// - `No`, no parentheses are expected.
|
||||
/// - `Maybe`, consume the `)` token if it's present.
|
||||
///
|
||||
/// The contained start position in each variant is used to determine the range
|
||||
/// of the generator expression.
|
||||
/// The given `start` offset is the start of either the opening parenthesis if the generator is
|
||||
/// parenthesized or the first token of the expression.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/expressions.html#generator-expressions>
|
||||
pub(super) fn parse_generator_expression(
|
||||
&mut self,
|
||||
element: Expr,
|
||||
in_parentheses: GeneratorExpressionInParentheses,
|
||||
start: TextSize,
|
||||
parenthesized: Parenthesized,
|
||||
) -> ast::ExprGenerator {
|
||||
let generators = self.parse_generators();
|
||||
|
||||
let (parenthesized, start) = match in_parentheses {
|
||||
GeneratorExpressionInParentheses::Yes(lpar_start) => {
|
||||
self.expect(TokenKind::Rpar);
|
||||
(true, lpar_start)
|
||||
}
|
||||
GeneratorExpressionInParentheses::No(expr_start) => (false, expr_start),
|
||||
GeneratorExpressionInParentheses::Maybe {
|
||||
lpar_start,
|
||||
expr_start,
|
||||
} => {
|
||||
if self.eat(TokenKind::Rpar) {
|
||||
(true, lpar_start)
|
||||
} else {
|
||||
(false, expr_start)
|
||||
}
|
||||
}
|
||||
};
|
||||
if parenthesized.is_yes() {
|
||||
self.expect(TokenKind::Rpar);
|
||||
}
|
||||
|
||||
ast::ExprGenerator {
|
||||
elt: Box::new(element),
|
||||
generators,
|
||||
range: self.node_range(start),
|
||||
parenthesized,
|
||||
parenthesized: parenthesized.is_yes(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2472,26 +2455,6 @@ impl From<Operator> for OperatorPrecedence {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum GeneratorExpressionInParentheses {
|
||||
/// The generator expression is in parentheses. The given [`TextSize`] is the
|
||||
/// start of the left parenthesis. E.g., `(x for x in range(10))`.
|
||||
Yes(TextSize),
|
||||
|
||||
/// The generator expression is not in parentheses. The given [`TextSize`] is the
|
||||
/// start of the expression. E.g., `x for x in range(10)`.
|
||||
No(TextSize),
|
||||
|
||||
/// The generator expression may or may not be in parentheses. The given [`TextSize`]s
|
||||
/// are the start of the left parenthesis and the start of the expression, respectively.
|
||||
Maybe {
|
||||
/// The start of the left parenthesis.
|
||||
lpar_start: TextSize,
|
||||
/// The start of the expression.
|
||||
expr_start: TextSize,
|
||||
},
|
||||
}
|
||||
|
||||
/// Represents the precedence used for parsing the value part of a starred expression.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum StarredExpressionPrecedence {
|
||||
|
|
|
|||
|
|
@ -705,16 +705,6 @@ enum WithItemKind {
|
|||
/// The parentheses belongs to the context expression.
|
||||
ParenthesizedExpression,
|
||||
|
||||
/// A list of `with` items that has only one item which is a parenthesized
|
||||
/// generator expression.
|
||||
///
|
||||
/// ```python
|
||||
/// with (x for x in range(10)): ...
|
||||
/// ```
|
||||
///
|
||||
/// The parentheses belongs to the generator expression.
|
||||
SingleParenthesizedGeneratorExpression,
|
||||
|
||||
/// The `with` items aren't parenthesized in any way.
|
||||
///
|
||||
/// ```python
|
||||
|
|
@ -732,20 +722,15 @@ impl WithItemKind {
|
|||
const fn list_terminator(self) -> TokenKind {
|
||||
match self {
|
||||
WithItemKind::Parenthesized => TokenKind::Rpar,
|
||||
WithItemKind::Unparenthesized
|
||||
| WithItemKind::ParenthesizedExpression
|
||||
| WithItemKind::SingleParenthesizedGeneratorExpression => TokenKind::Colon,
|
||||
WithItemKind::Unparenthesized | WithItemKind::ParenthesizedExpression => {
|
||||
TokenKind::Colon
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the `with` item is a parenthesized expression i.e., the
|
||||
/// parentheses belong to the context expression.
|
||||
const fn is_parenthesized_expression(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
WithItemKind::ParenthesizedExpression
|
||||
| WithItemKind::SingleParenthesizedGeneratorExpression
|
||||
)
|
||||
/// Returns `true` if the with items are parenthesized.
|
||||
const fn is_parenthesized(self) -> bool {
|
||||
matches!(self, WithItemKind::Parenthesized)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1172,7 +1157,6 @@ bitflags! {
|
|||
const LAMBDA_PARAMETERS = 1 << 24;
|
||||
const WITH_ITEMS_PARENTHESIZED = 1 << 25;
|
||||
const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
|
||||
const WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION = 1 << 27;
|
||||
const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
|
||||
const F_STRING_ELEMENTS = 1 << 29;
|
||||
}
|
||||
|
|
@ -1225,9 +1209,6 @@ impl RecoveryContext {
|
|||
WithItemKind::ParenthesizedExpression => {
|
||||
RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION
|
||||
}
|
||||
WithItemKind::SingleParenthesizedGeneratorExpression => {
|
||||
RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION
|
||||
}
|
||||
WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
|
||||
},
|
||||
RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
|
||||
|
|
@ -1294,9 +1275,6 @@ impl RecoveryContext {
|
|||
RecoveryContext::WITH_ITEMS_PARENTHESIZED_EXPRESSION => {
|
||||
RecoveryContextKind::WithItems(WithItemKind::ParenthesizedExpression)
|
||||
}
|
||||
RecoveryContext::WITH_ITEMS_SINGLE_PARENTHESIZED_GENERATOR_EXPRESSION => {
|
||||
RecoveryContextKind::WithItems(WithItemKind::SingleParenthesizedGeneratorExpression)
|
||||
}
|
||||
RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
|
||||
RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use ruff_python_ast::{
|
|||
use ruff_text_size::{Ranged, TextSize};
|
||||
|
||||
use crate::lexer::TokenValue;
|
||||
use crate::parser::expression::{GeneratorExpressionInParentheses, ParsedExpr, EXPR_SET};
|
||||
use crate::parser::expression::{ParsedExpr, EXPR_SET};
|
||||
use crate::parser::progress::ParserProgress;
|
||||
use crate::parser::{
|
||||
helpers, FunctionKind, Parser, RecoveryContext, RecoveryContextKind, WithItemKind,
|
||||
|
|
@ -17,7 +17,7 @@ use crate::parser::{
|
|||
use crate::token_set::TokenSet;
|
||||
use crate::{Mode, ParseErrorType, TokenKind};
|
||||
|
||||
use super::expression::{ExpressionContext, OperatorPrecedence};
|
||||
use super::expression::ExpressionContext;
|
||||
use super::Parenthesized;
|
||||
|
||||
/// Tokens that represent compound statements.
|
||||
|
|
@ -1886,11 +1886,8 @@ impl<'src> Parser<'src> {
|
|||
|
||||
/// Parses a list of with items.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-with_stmt_contents>
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#the-with-statement>
|
||||
fn parse_with_items(&mut self) -> Vec<WithItem> {
|
||||
let start = self.node_start();
|
||||
let mut items = vec![];
|
||||
|
||||
if !self.at_expr() {
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
|
|
@ -1898,315 +1895,20 @@ impl<'src> Parser<'src> {
|
|||
),
|
||||
self.current_token_range(),
|
||||
);
|
||||
return items;
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let with_item_kind = if self.eat(TokenKind::Lpar) {
|
||||
self.parse_parenthesized_with_items(start, &mut items)
|
||||
} else {
|
||||
WithItemKind::Unparenthesized
|
||||
};
|
||||
|
||||
if with_item_kind.is_parenthesized_expression() {
|
||||
// The trailing comma is optional because (1) they aren't allowed in parenthesized
|
||||
// expression context and, (2) We need to raise the correct error if they're present.
|
||||
//
|
||||
// Consider the following three examples:
|
||||
//
|
||||
// ```python
|
||||
// with (item1, item2): ... # (1)
|
||||
// with (item1, item2),: ... # (2)
|
||||
// with (item1, item2), item3,: ... # (3)
|
||||
// ```
|
||||
//
|
||||
// Here, (1) is valid and represents a parenthesized with items while (2) and (3)
|
||||
// are invalid as they are parenthesized expression. Example (3) will raise an error
|
||||
// stating that a trailing comma isn't allowed, while (2) will raise an "expected an
|
||||
// expression" error.
|
||||
//
|
||||
// The reason that (2) expects an expression is because if it raised an error
|
||||
// similar to (3), we would be suggesting to remove the trailing comma, which would
|
||||
// make it a parenthesized with items. This would contradict our original assumption
|
||||
// that it's a parenthesized expression.
|
||||
//
|
||||
// However, for (3), the error is being raised by the list parsing logic and if the
|
||||
// trailing comma is removed, it still remains a parenthesized expression, so it's
|
||||
// fine to raise the error.
|
||||
if self.eat(TokenKind::Comma) && !self.at_expr() {
|
||||
self.add_error(
|
||||
ParseErrorType::ExpectedExpression,
|
||||
self.current_token_range(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// This call is a no-op if the with items are parenthesized as all of them
|
||||
// have already been parsed.
|
||||
self.parse_comma_separated_list(RecoveryContextKind::WithItems(with_item_kind), |parser| {
|
||||
items.push(parser.parse_with_item(WithItemParsingState::Regular).item);
|
||||
});
|
||||
|
||||
if with_item_kind == WithItemKind::Parenthesized {
|
||||
self.expect(TokenKind::Rpar);
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
/// Parse the with items coming after an ambiguous `(` token.
|
||||
///
|
||||
/// To understand the ambiguity, consider the following example:
|
||||
///
|
||||
/// ```python
|
||||
/// with (item1, item2): ... # (1)
|
||||
/// with (item1, item2) as f: ... # (2)
|
||||
/// ```
|
||||
///
|
||||
/// When the parser is at the `(` token after the `with` keyword, it doesn't
|
||||
/// know if it's used to parenthesize the with items or if it's part of a
|
||||
/// parenthesized expression of the first with item. The challenge here is
|
||||
/// that until the parser sees the matching `)` token, it can't resolve the
|
||||
/// ambiguity. This requires infinite lookahead.
|
||||
///
|
||||
/// This method resolves the ambiguity by parsing the with items assuming that
|
||||
/// it's a parenthesized with items. Then, once it finds the matching `)`, it
|
||||
/// checks if the assumption still holds true. If it doesn't, then it combines
|
||||
/// the parsed with items into a single with item with an appropriate expression.
|
||||
///
|
||||
/// The return value is the kind of with items parsed. Note that there could
|
||||
/// still be other with items which needs to be parsed as this method stops
|
||||
/// when the matching `)` is found.
|
||||
fn parse_parenthesized_with_items(
|
||||
&mut self,
|
||||
start: TextSize,
|
||||
items: &mut Vec<WithItem>,
|
||||
) -> WithItemKind {
|
||||
// We'll start with the assumption that the with items are parenthesized.
|
||||
let mut with_item_kind = WithItemKind::Parenthesized;
|
||||
|
||||
// Keep track of certain properties to determine if the with items are
|
||||
// parenthesized or if it's a parenthesized expression. Refer to their
|
||||
// usage for examples and explanation.
|
||||
let mut has_trailing_comma = false;
|
||||
let mut has_optional_vars = false;
|
||||
|
||||
// Start with parsing the first with item after an ambiguous `(` token
|
||||
// with the start offset.
|
||||
let mut state = WithItemParsingState::AmbiguousLparFirstItem(start);
|
||||
|
||||
let mut parsed_with_items = vec![];
|
||||
let mut progress = ParserProgress::default();
|
||||
|
||||
loop {
|
||||
progress.assert_progressing(self);
|
||||
|
||||
// We stop at the first `)` found. Any nested parentheses will be
|
||||
// consumed by the with item parsing. This check needs to be done
|
||||
// first in case there are no with items. For example,
|
||||
//
|
||||
// ```python
|
||||
// with (): ...
|
||||
// with () as x: ...
|
||||
// ```
|
||||
if self.at(TokenKind::Rpar) {
|
||||
break;
|
||||
}
|
||||
|
||||
let parsed_with_item = self.parse_with_item(state);
|
||||
|
||||
if parsed_with_item.item.context_expr.is_generator_expr()
|
||||
&& parsed_with_item.used_ambiguous_lpar
|
||||
{
|
||||
// For generator expressions, it's a bit tricky. We need to check if parsing
|
||||
// a generator expression has used the ambiguous `(` token. This is the case
|
||||
// for a parenthesized generator expression which is using the ambiguous `(`
|
||||
// as the start of the generator expression. For example:
|
||||
//
|
||||
// ```python
|
||||
// with (x for x in range(10)): ...
|
||||
// # ^
|
||||
// # Consumed by `parse_with_item`
|
||||
// ```
|
||||
//
|
||||
// This is only allowed if it's the first with item which is made sure by the
|
||||
// `with_item_parsing` state.
|
||||
with_item_kind = WithItemKind::SingleParenthesizedGeneratorExpression;
|
||||
parsed_with_items.push(parsed_with_item);
|
||||
break;
|
||||
}
|
||||
|
||||
has_optional_vars |= parsed_with_item.item.optional_vars.is_some();
|
||||
|
||||
parsed_with_items.push(parsed_with_item);
|
||||
|
||||
has_trailing_comma = self.eat(TokenKind::Comma);
|
||||
if !has_trailing_comma {
|
||||
break;
|
||||
}
|
||||
|
||||
// Update the with item parsing to indicate that we're no longer
|
||||
// parsing the first with item, but we haven't yet found the `)` to
|
||||
// the corresponding ambiguous `(`.
|
||||
state = WithItemParsingState::AmbiguousLparRest;
|
||||
}
|
||||
|
||||
// Check if our assumption is incorrect and it's actually a parenthesized
|
||||
// expression.
|
||||
if !with_item_kind.is_parenthesized_expression() && self.at(TokenKind::Rpar) {
|
||||
if has_optional_vars {
|
||||
// If any of the with item has optional variables, then our assumption is
|
||||
// correct and it is a parenthesized with items. Now, we need to restrict
|
||||
// the grammar for a with item's context expression which is:
|
||||
//
|
||||
// with_item: expression ...
|
||||
//
|
||||
// So, named, starred and yield expressions not allowed.
|
||||
for parsed_with_item in &parsed_with_items {
|
||||
// Parentheses resets the precedence.
|
||||
if parsed_with_item.is_parenthesized {
|
||||
continue;
|
||||
}
|
||||
let err = match parsed_with_item.item.context_expr {
|
||||
Expr::Named(_) => ParseErrorType::UnparenthesizedNamedExpression,
|
||||
Expr::Starred(_) => ParseErrorType::InvalidStarredExpressionUsage,
|
||||
Expr::Yield(_) | Expr::YieldFrom(_) => {
|
||||
ParseErrorType::InvalidYieldExpressionUsage
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
self.add_error(err, &parsed_with_item.item.context_expr);
|
||||
}
|
||||
} else if self.peek() == TokenKind::Colon {
|
||||
// Here, the parser is at a `)` followed by a `:`.
|
||||
if parsed_with_items.is_empty() {
|
||||
// No with items, treat it as a parenthesized expression to
|
||||
// create an empty tuple expression.
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
} else {
|
||||
// These expressions, if unparenthesized, are only allowed if it's
|
||||
// a parenthesized expression and none of the with items have an
|
||||
// optional variable.
|
||||
if parsed_with_items.iter().any(|parsed_with_item| {
|
||||
!parsed_with_item.is_parenthesized
|
||||
&& matches!(
|
||||
parsed_with_item.item.context_expr,
|
||||
Expr::Named(_)
|
||||
| Expr::Starred(_)
|
||||
| Expr::Yield(_)
|
||||
| Expr::YieldFrom(_)
|
||||
)
|
||||
}) {
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For any other token followed by `)`, if any of the items has
|
||||
// an optional variables (`as ...`), then our assumption is correct.
|
||||
// Otherwise, treat it as a parenthesized expression. For example:
|
||||
//
|
||||
// ```python
|
||||
// with (item1, item2 as f): ...
|
||||
// ```
|
||||
//
|
||||
// This also helps in raising the correct syntax error for the
|
||||
// following case:
|
||||
// ```python
|
||||
// with (item1, item2 as f) as x: ...
|
||||
// # ^^
|
||||
// # Expecting `:` but got `as`
|
||||
// ```
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
}
|
||||
}
|
||||
|
||||
if with_item_kind == WithItemKind::Parenthesized && !self.at(TokenKind::Rpar) {
|
||||
// test_err with_items_parenthesized_missing_comma
|
||||
// with (item1 item2): ...
|
||||
// with (item1 as f1 item2): ...
|
||||
// with (item1, item2 item3, item4): ...
|
||||
// with (item1, item2 as f1 item3, item4): ...
|
||||
// with (item1, item2: ...
|
||||
self.expect(TokenKind::Comma);
|
||||
}
|
||||
|
||||
// Transform the items if it's a parenthesized expression.
|
||||
if with_item_kind.is_parenthesized_expression() {
|
||||
// The generator expression has already consumed the `)`, so avoid
|
||||
// expecting it again.
|
||||
if with_item_kind != WithItemKind::SingleParenthesizedGeneratorExpression {
|
||||
if self.at(TokenKind::Lpar) {
|
||||
if let Some(items) = self.try_parse_parenthesized_with_items() {
|
||||
self.expect(TokenKind::Rpar);
|
||||
}
|
||||
|
||||
let mut lhs = if parsed_with_items.len() == 1 && !has_trailing_comma {
|
||||
// SAFETY: We've checked that `items` has only one item.
|
||||
let expr = parsed_with_items.pop().unwrap().item.context_expr;
|
||||
|
||||
// Here, we know that it's a parenthesized expression so the expression
|
||||
// should be checked against the grammar rule which is:
|
||||
//
|
||||
// group: (yield_expr | named_expression)
|
||||
//
|
||||
// So, no starred expression allowed.
|
||||
if expr.is_starred_expr() {
|
||||
self.add_error(ParseErrorType::InvalidStarredExpressionUsage, &expr);
|
||||
}
|
||||
expr
|
||||
items
|
||||
} else {
|
||||
let mut elts = Vec::with_capacity(parsed_with_items.len());
|
||||
|
||||
// Here, we know that it's a tuple expression so each expression should
|
||||
// be checked against the tuple element grammar rule which:
|
||||
//
|
||||
// tuple: '(' [ star_named_expression ',' [star_named_expressions] ] ')'
|
||||
//
|
||||
// So, no yield expressions allowed.
|
||||
for expr in parsed_with_items
|
||||
.drain(..)
|
||||
.map(|parsed_with_item| parsed_with_item.item.context_expr)
|
||||
{
|
||||
if matches!(expr, Expr::Yield(_) | Expr::YieldFrom(_)) {
|
||||
self.add_error(ParseErrorType::InvalidYieldExpressionUsage, &expr);
|
||||
}
|
||||
elts.push(expr);
|
||||
}
|
||||
|
||||
Expr::Tuple(ast::ExprTuple {
|
||||
range: self.node_range(start),
|
||||
elts,
|
||||
ctx: ExprContext::Load,
|
||||
parenthesized: true,
|
||||
})
|
||||
};
|
||||
|
||||
// Remember that the expression is parenthesized and the parser has just
|
||||
// consumed the `)` token. We need to check for any possible postfix
|
||||
// expressions. For example:
|
||||
//
|
||||
// ```python
|
||||
// with (foo)(): ...
|
||||
// # ^
|
||||
//
|
||||
// with (1, 2)[0]: ...
|
||||
// # ^
|
||||
//
|
||||
// with (foo.bar).baz: ...
|
||||
// # ^
|
||||
// ```
|
||||
//
|
||||
// The reason being that the opening parenthesis is ambiguous and isn't
|
||||
// considered when parsing the with item in the case. So, the parser
|
||||
// stops when it sees the `)` token and doesn't check for any postfix
|
||||
// expressions.
|
||||
lhs = self.parse_postfix_expression(lhs, start);
|
||||
|
||||
let context_expr = if self.at(TokenKind::If) {
|
||||
// test_ok ambiguous_lpar_with_items_if_expr
|
||||
// with (x) if True else y: ...
|
||||
// with (x for x in iter) if True else y: ...
|
||||
// with (x async for x in iter) if True else y: ...
|
||||
// with (x)[0] if True else y: ...
|
||||
Expr::If(self.parse_if_expression(lhs, start))
|
||||
} else {
|
||||
|
||||
// test_ok ambiguous_lpar_with_items_binary_expr
|
||||
// # It doesn't matter what's inside the parentheses, these tests need to make sure
|
||||
// # all binary expressions parses correctly.
|
||||
|
|
@ -2218,29 +1920,147 @@ impl<'src> Parser<'src> {
|
|||
// with (a | b) << c | d: ...
|
||||
// # Postfix should still be parsed first
|
||||
// with (a)[0] + b * c: ...
|
||||
self.parse_binary_expression_or_higher_recursive(
|
||||
lhs.into(),
|
||||
OperatorPrecedence::Initial,
|
||||
ExpressionContext::default(),
|
||||
start,
|
||||
self.parse_comma_separated_list_into_vec(
|
||||
RecoveryContextKind::WithItems(WithItemKind::ParenthesizedExpression),
|
||||
|p| p.parse_with_item(WithItemParsingState::Regular).item,
|
||||
)
|
||||
.expr
|
||||
};
|
||||
|
||||
let optional_vars = self
|
||||
.at(TokenKind::As)
|
||||
.then(|| Box::new(self.parse_with_item_optional_vars().expr));
|
||||
|
||||
items.push(ast::WithItem {
|
||||
range: self.node_range(start),
|
||||
context_expr,
|
||||
optional_vars,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
items.extend(parsed_with_items.drain(..).map(|item| item.item));
|
||||
self.parse_comma_separated_list_into_vec(
|
||||
RecoveryContextKind::WithItems(WithItemKind::Unparenthesized),
|
||||
|p| p.parse_with_item(WithItemParsingState::Regular).item,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Try parsing with-items coming after an ambiguous `(` token.
|
||||
///
|
||||
/// To understand the ambiguity, consider the following example:
|
||||
///
|
||||
/// ```python
|
||||
/// with (item1, item2): ... # Parenthesized with items
|
||||
/// with (item1, item2) as f: ... # Parenthesized expression
|
||||
/// ```
|
||||
///
|
||||
/// When the parser is at the `(` token after the `with` keyword, it doesn't know if `(` is
|
||||
/// used to parenthesize the with items or if it's part of a parenthesized expression of the
|
||||
/// first with item. The challenge here is that until the parser sees the matching `)` token,
|
||||
/// it can't resolve the ambiguity.
|
||||
///
|
||||
/// This method resolves the ambiguity using speculative parsing. It starts with an assumption
|
||||
/// that it's a parenthesized with items. Then, once it finds the matching `)`, it checks if
|
||||
/// the assumption still holds true. If the initial assumption was correct, this will return
|
||||
/// the parsed with items. Otherwise, rewind the parser back to the starting `(` token,
|
||||
/// returning [`None`].
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// If the parser isn't positioned at a `(` token.
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-with_stmt_contents>
|
||||
fn try_parse_parenthesized_with_items(&mut self) -> Option<Vec<WithItem>> {
|
||||
let checkpoint = self.checkpoint();
|
||||
|
||||
// We'll start with the assumption that the with items are parenthesized.
|
||||
let mut with_item_kind = WithItemKind::Parenthesized;
|
||||
|
||||
self.bump(TokenKind::Lpar);
|
||||
|
||||
let mut parsed_with_items = vec![];
|
||||
let mut has_optional_vars = false;
|
||||
|
||||
// test_err with_items_parenthesized_missing_comma
|
||||
// with (item1 item2): ...
|
||||
// with (item1 as f1 item2): ...
|
||||
// with (item1, item2 item3, item4): ...
|
||||
// with (item1, item2 as f1 item3, item4): ...
|
||||
// with (item1, item2: ...
|
||||
self.parse_comma_separated_list(RecoveryContextKind::WithItems(with_item_kind), |p| {
|
||||
let parsed_with_item = p.parse_with_item(WithItemParsingState::Speculative);
|
||||
has_optional_vars |= parsed_with_item.item.optional_vars.is_some();
|
||||
parsed_with_items.push(parsed_with_item);
|
||||
});
|
||||
|
||||
// Check if our assumption is incorrect and it's actually a parenthesized expression.
|
||||
if has_optional_vars {
|
||||
// If any of the with item has optional variables, then our assumption is correct
|
||||
// and it is a parenthesized with items. Now, we need to restrict the grammar for a
|
||||
// with item's context expression which is:
|
||||
//
|
||||
// with_item: expression ...
|
||||
//
|
||||
// So, named, starred and yield expressions not allowed.
|
||||
for parsed_with_item in &parsed_with_items {
|
||||
if parsed_with_item.is_parenthesized {
|
||||
// Parentheses resets the precedence.
|
||||
continue;
|
||||
}
|
||||
let error = match parsed_with_item.item.context_expr {
|
||||
Expr::Named(_) => ParseErrorType::UnparenthesizedNamedExpression,
|
||||
Expr::Starred(_) => ParseErrorType::InvalidStarredExpressionUsage,
|
||||
Expr::Yield(_) | Expr::YieldFrom(_) => {
|
||||
ParseErrorType::InvalidYieldExpressionUsage
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
self.add_error(error, &parsed_with_item.item.context_expr);
|
||||
}
|
||||
} else if self.at(TokenKind::Rpar)
|
||||
// test_err with_items_parenthesized_missing_colon
|
||||
// # `)` followed by a newline
|
||||
// with (item1, item2)
|
||||
// pass
|
||||
&& matches!(self.peek(), TokenKind::Colon | TokenKind::Newline)
|
||||
{
|
||||
if parsed_with_items.is_empty() {
|
||||
// No with items, treat it as a parenthesized expression to create an empty
|
||||
// tuple expression.
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
} else {
|
||||
// These expressions, if unparenthesized, are only allowed if it's a
|
||||
// parenthesized expression and none of the with items have an optional
|
||||
// variable.
|
||||
if parsed_with_items.iter().any(|parsed_with_item| {
|
||||
!parsed_with_item.is_parenthesized
|
||||
&& matches!(
|
||||
parsed_with_item.item.context_expr,
|
||||
Expr::Named(_) | Expr::Starred(_) | Expr::Yield(_) | Expr::YieldFrom(_)
|
||||
)
|
||||
}) {
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For any other token followed by `)`, if any of the items has an optional
|
||||
// variables (`as ...`), then our assumption is correct. Otherwise, treat
|
||||
// it as a parenthesized expression. For example:
|
||||
//
|
||||
// ```python
|
||||
// with (item1, item2 as f): ...
|
||||
// ```
|
||||
//
|
||||
// This also helps in raising the correct syntax error for the following
|
||||
// case:
|
||||
// ```python
|
||||
// with (item1, item2 as f) as x: ...
|
||||
// # ^^
|
||||
// # Expecting `:` but got `as`
|
||||
// ```
|
||||
with_item_kind = WithItemKind::ParenthesizedExpression;
|
||||
}
|
||||
|
||||
with_item_kind
|
||||
if with_item_kind.is_parenthesized() {
|
||||
Some(
|
||||
parsed_with_items
|
||||
.into_iter()
|
||||
.map(|parsed_with_item| parsed_with_item.item)
|
||||
.collect(),
|
||||
)
|
||||
} else {
|
||||
self.rewind(checkpoint);
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a single `with` item.
|
||||
|
|
@ -2249,93 +2069,33 @@ impl<'src> Parser<'src> {
|
|||
fn parse_with_item(&mut self, state: WithItemParsingState) -> ParsedWithItem {
|
||||
let start = self.node_start();
|
||||
|
||||
let mut used_ambiguous_lpar = false;
|
||||
|
||||
// The grammar for the context expression of a with item depends on the state
|
||||
// of with item parsing.
|
||||
let context_expr = if state.is_ambiguous_lpar() {
|
||||
// If it's in an ambiguous state, the parenthesis (`(`) could be part of any
|
||||
// of the following expression:
|
||||
//
|
||||
// Tuple expression - star_named_expression
|
||||
// Generator expression - named_expression
|
||||
// Parenthesized expression - (yield_expr | named_expression)
|
||||
// Parenthesized with items - expression
|
||||
//
|
||||
// Here, the right side specifies the grammar for an element corresponding
|
||||
// to the expression mentioned in the left side.
|
||||
//
|
||||
// So, the grammar used should be able to parse an element belonging to any
|
||||
// of the above expression. At a later point, once the parser understands
|
||||
// where the parenthesis belongs to, it'll validate and report errors for
|
||||
// any invalid expression usage.
|
||||
//
|
||||
// Thus, we can conclude that the grammar used should be:
|
||||
// (yield_expr | star_named_expression)
|
||||
let parsed_expr = self
|
||||
.parse_named_expression_or_higher(ExpressionContext::yield_or_starred_bitwise_or());
|
||||
|
||||
if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) {
|
||||
if parsed_expr.is_unparenthesized_starred_expr() {
|
||||
self.add_error(
|
||||
ParseErrorType::IterableUnpackingInComprehension,
|
||||
&parsed_expr,
|
||||
);
|
||||
}
|
||||
|
||||
let generator_expr =
|
||||
if let WithItemParsingState::AmbiguousLparFirstItem(lpar_start) = state {
|
||||
// The parser is at the first with item after the ambiguous `(` token.
|
||||
// For example:
|
||||
//
|
||||
// ```python
|
||||
// with (x for x in range(10)): ...
|
||||
// with (x for x in range(10)), item: ...
|
||||
// ```
|
||||
let generator_expr = self.parse_generator_expression(
|
||||
parsed_expr.expr,
|
||||
GeneratorExpressionInParentheses::Maybe {
|
||||
lpar_start,
|
||||
expr_start: start,
|
||||
},
|
||||
);
|
||||
used_ambiguous_lpar = generator_expr.parenthesized;
|
||||
generator_expr
|
||||
} else {
|
||||
// For better error recovery. We would not take this path if the
|
||||
// expression was parenthesized as it would be parsed as a generator
|
||||
// expression by `parse_conditional_expression_or_higher`.
|
||||
//
|
||||
// ```python
|
||||
// # This path will be taken for
|
||||
// with (item, x for x in range(10)): ...
|
||||
//
|
||||
// # This path will not be taken for
|
||||
// with (item, (x for x in range(10))): ...
|
||||
// ```
|
||||
self.parse_generator_expression(
|
||||
parsed_expr.expr,
|
||||
GeneratorExpressionInParentheses::No(start),
|
||||
)
|
||||
};
|
||||
|
||||
if !generator_expr.parenthesized {
|
||||
self.add_error(
|
||||
ParseErrorType::OtherError(
|
||||
"Unparenthesized generator expression cannot be used here".to_string(),
|
||||
),
|
||||
generator_expr.range(),
|
||||
);
|
||||
}
|
||||
|
||||
Expr::Generator(generator_expr).into()
|
||||
} else {
|
||||
parsed_expr
|
||||
let context_expr = match state {
|
||||
WithItemParsingState::Speculative => {
|
||||
// If it's in a speculative state, the parenthesis (`(`) could be part of any of the
|
||||
// following expression:
|
||||
//
|
||||
// Tuple expression - star_named_expression
|
||||
// Generator expression - named_expression
|
||||
// Parenthesized expression - (yield_expr | named_expression)
|
||||
// Parenthesized with items - expression
|
||||
//
|
||||
// Here, the right side specifies the grammar for an element corresponding to the
|
||||
// expression mentioned in the left side.
|
||||
//
|
||||
// So, the grammar used should be able to parse an element belonging to any of the
|
||||
// above expression. At a later point, once the parser understands where the
|
||||
// parenthesis belongs to, it'll validate and report errors for any invalid expression
|
||||
// usage.
|
||||
//
|
||||
// Thus, we can conclude that the grammar used should be:
|
||||
// (yield_expr | star_named_expression)
|
||||
self.parse_named_expression_or_higher(
|
||||
ExpressionContext::yield_or_starred_bitwise_or(),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// If it's not in an ambiguous state, then the grammar of the with item
|
||||
// should be used which is `expression`.
|
||||
self.parse_conditional_expression_or_higher()
|
||||
WithItemParsingState::Regular => self.parse_conditional_expression_or_higher(),
|
||||
};
|
||||
|
||||
let optional_vars = self
|
||||
|
|
@ -2344,7 +2104,6 @@ impl<'src> Parser<'src> {
|
|||
|
||||
ParsedWithItem {
|
||||
is_parenthesized: context_expr.is_parenthesized,
|
||||
used_ambiguous_lpar,
|
||||
item: ast::WithItem {
|
||||
range: self.node_range(start),
|
||||
context_expr: context_expr.expr,
|
||||
|
|
@ -3768,46 +3527,19 @@ enum MatchTokenKind {
|
|||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum WithItemParsingState {
|
||||
/// The parser is currently parsing a with item without any ambiguity.
|
||||
/// Parsing the with items without any ambiguity.
|
||||
Regular,
|
||||
|
||||
/// The parser is currently parsing the first with item after an ambiguous
|
||||
/// left parenthesis. The contained offset is the start of the left parenthesis.
|
||||
///
|
||||
/// ```python
|
||||
/// with (item1, item2): ...
|
||||
/// ```
|
||||
///
|
||||
/// The parser is at the start of `item1`.
|
||||
AmbiguousLparFirstItem(TextSize),
|
||||
|
||||
/// The parser is currently parsing one of the with items after an ambiguous
|
||||
/// left parenthesis, but not the first one.
|
||||
///
|
||||
/// ```python
|
||||
/// with (item1, item2, item3): ...
|
||||
/// ```
|
||||
///
|
||||
/// The parser could be at the start of `item2` or `item3`, but not `item1`.
|
||||
AmbiguousLparRest,
|
||||
}
|
||||
|
||||
impl WithItemParsingState {
|
||||
const fn is_ambiguous_lpar(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::AmbiguousLparFirstItem(_) | Self::AmbiguousLparRest
|
||||
)
|
||||
}
|
||||
/// Parsing the with items in a speculative mode.
|
||||
Speculative,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ParsedWithItem {
|
||||
/// The contained with item.
|
||||
item: WithItem,
|
||||
/// If the context expression of the item is parenthesized.
|
||||
is_parenthesized: bool,
|
||||
/// If the parsing used the ambiguous left parenthesis.
|
||||
used_ambiguous_lpar: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
|
|
|
|||
|
|
@ -375,10 +375,10 @@ Module(
|
|||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 364..384,
|
||||
range: 363..384,
|
||||
context_expr: Generator(
|
||||
ExprGenerator {
|
||||
range: 364..384,
|
||||
range: 363..384,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 364..365,
|
||||
|
|
@ -426,7 +426,7 @@ Module(
|
|||
is_async: false,
|
||||
},
|
||||
],
|
||||
parenthesized: false,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
|
|
@ -459,90 +459,89 @@ Module(
|
|||
),
|
||||
With(
|
||||
StmtWith {
|
||||
range: 397..435,
|
||||
range: 397..410,
|
||||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 403..407,
|
||||
context_expr: Name(
|
||||
range: 402..410,
|
||||
context_expr: Tuple(
|
||||
ExprTuple {
|
||||
range: 402..410,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 403..407,
|
||||
id: "item",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
Name(
|
||||
ExprName {
|
||||
range: 409..410,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
],
|
||||
ctx: Load,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [],
|
||||
},
|
||||
),
|
||||
For(
|
||||
StmtFor {
|
||||
range: 411..429,
|
||||
is_async: false,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 415..416,
|
||||
id: "x",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 420..429,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 403..407,
|
||||
id: "item",
|
||||
range: 420..425,
|
||||
id: "range",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 409..429,
|
||||
context_expr: Generator(
|
||||
ExprGenerator {
|
||||
range: 409..429,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 409..410,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
arguments: Arguments {
|
||||
range: 425..429,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 426..428,
|
||||
value: Int(
|
||||
10,
|
||||
),
|
||||
},
|
||||
),
|
||||
generators: [
|
||||
Comprehension {
|
||||
range: 411..429,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 415..416,
|
||||
id: "x",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 420..429,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 420..425,
|
||||
id: "range",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
arguments: Arguments {
|
||||
range: 425..429,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 426..428,
|
||||
value: Int(
|
||||
10,
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
keywords: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
ifs: [],
|
||||
is_async: false,
|
||||
},
|
||||
],
|
||||
parenthesized: false,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 432..435,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 432..435,
|
||||
},
|
||||
),
|
||||
],
|
||||
keywords: [],
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
body: [],
|
||||
orelse: [],
|
||||
},
|
||||
),
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 432..435,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 432..435,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
With(
|
||||
|
|
@ -588,10 +587,10 @@ Module(
|
|||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 523..539,
|
||||
range: 522..539,
|
||||
context_expr: Generator(
|
||||
ExprGenerator {
|
||||
range: 523..539,
|
||||
range: 522..539,
|
||||
elt: Starred(
|
||||
ExprStarred {
|
||||
range: 523..525,
|
||||
|
|
@ -626,7 +625,7 @@ Module(
|
|||
is_async: false,
|
||||
},
|
||||
],
|
||||
parenthesized: false,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
|
|
@ -659,88 +658,92 @@ Module(
|
|||
),
|
||||
With(
|
||||
StmtWith {
|
||||
range: 552..594,
|
||||
range: 552..567,
|
||||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 558..563,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 558..563,
|
||||
id: "item1",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 565..581,
|
||||
context_expr: Generator(
|
||||
ExprGenerator {
|
||||
range: 565..581,
|
||||
elt: Starred(
|
||||
ExprStarred {
|
||||
range: 565..567,
|
||||
value: Name(
|
||||
ExprName {
|
||||
range: 566..567,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
generators: [
|
||||
Comprehension {
|
||||
range: 568..581,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 572..573,
|
||||
id: "x",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Name(
|
||||
ExprName {
|
||||
range: 577..581,
|
||||
id: "iter",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
ifs: [],
|
||||
is_async: false,
|
||||
},
|
||||
range: 557..567,
|
||||
context_expr: Tuple(
|
||||
ExprTuple {
|
||||
range: 557..567,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 558..563,
|
||||
id: "item1",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
Starred(
|
||||
ExprStarred {
|
||||
range: 565..567,
|
||||
value: Name(
|
||||
ExprName {
|
||||
range: 566..567,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
],
|
||||
parenthesized: false,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 583..588,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 583..588,
|
||||
id: "item2",
|
||||
ctx: Load,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 591..594,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 591..594,
|
||||
body: [],
|
||||
},
|
||||
),
|
||||
For(
|
||||
StmtFor {
|
||||
range: 568..588,
|
||||
is_async: false,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 572..573,
|
||||
id: "x",
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Tuple(
|
||||
ExprTuple {
|
||||
range: 577..588,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 577..581,
|
||||
id: "iter",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
Name(
|
||||
ExprName {
|
||||
range: 583..588,
|
||||
id: "item2",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
],
|
||||
ctx: Load,
|
||||
parenthesized: false,
|
||||
},
|
||||
),
|
||||
body: [],
|
||||
orelse: [],
|
||||
},
|
||||
),
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 591..594,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 591..594,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
With(
|
||||
|
|
@ -1095,10 +1098,10 @@ Module(
|
|||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 751..771,
|
||||
range: 750..771,
|
||||
context_expr: Generator(
|
||||
ExprGenerator {
|
||||
range: 751..766,
|
||||
range: 750..766,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 751..752,
|
||||
|
|
@ -1127,7 +1130,7 @@ Module(
|
|||
is_async: false,
|
||||
},
|
||||
],
|
||||
parenthesized: false,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: Some(
|
||||
|
|
@ -1360,7 +1363,7 @@ Module(
|
|||
2 | # These cases should raise the correct syntax error and recover properly.
|
||||
3 |
|
||||
4 | with (item1, item2),: ...
|
||||
| ^ Syntax Error: Expected an expression
|
||||
| ^ Syntax Error: Trailing comma not allowed
|
||||
5 | with (item1, item2), as f: ...
|
||||
6 | with (item1, item2), item3,: ...
|
||||
|
|
||||
|
|
@ -1369,7 +1372,7 @@ Module(
|
|||
|
|
||||
4 | with (item1, item2),: ...
|
||||
5 | with (item1, item2), as f: ...
|
||||
| ^^ Syntax Error: Expected an expression
|
||||
| ^^ Syntax Error: Expected an expression or the end of the with item list
|
||||
6 | with (item1, item2), item3,: ...
|
||||
7 | with (*item): ...
|
||||
|
|
||||
|
|
@ -1429,7 +1432,16 @@ Module(
|
|||
9 | with (item := 10 as f): ...
|
||||
10 | with (item1, item2 := 10 as f): ...
|
||||
11 | with (x for x in range(10), item): ...
|
||||
| ^^^^^^^^^^^^^^^^^^^^ Syntax Error: Unparenthesized generator expression cannot be used here
|
||||
| ^ Syntax Error: Expected ')', found ','
|
||||
12 | with (item, x for x in range(10)): ...
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
9 | with (item := 10 as f): ...
|
||||
10 | with (item1, item2 := 10 as f): ...
|
||||
11 | with (x for x in range(10), item): ...
|
||||
| ^ Syntax Error: Expected ',', found ')'
|
||||
12 | with (item, x for x in range(10)): ...
|
||||
|
|
||||
|
||||
|
|
@ -1438,7 +1450,27 @@ Module(
|
|||
10 | with (item1, item2 := 10 as f): ...
|
||||
11 | with (x for x in range(10), item): ...
|
||||
12 | with (item, x for x in range(10)): ...
|
||||
| ^^^^^^^^^^^^^^^^^^^^ Syntax Error: Unparenthesized generator expression cannot be used here
|
||||
| ^^^ Syntax Error: Expected ',', found 'for'
|
||||
13 |
|
||||
14 | # Make sure the parser doesn't report the same error twice
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
10 | with (item1, item2 := 10 as f): ...
|
||||
11 | with (x for x in range(10), item): ...
|
||||
12 | with (item, x for x in range(10)): ...
|
||||
| ^ Syntax Error: Expected ':', found ')'
|
||||
13 |
|
||||
14 | # Make sure the parser doesn't report the same error twice
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
10 | with (item1, item2 := 10 as f): ...
|
||||
11 | with (x for x in range(10), item): ...
|
||||
12 | with (item, x for x in range(10)): ...
|
||||
| ^ Syntax Error: Expected a statement
|
||||
13 |
|
||||
14 | # Make sure the parser doesn't report the same error twice
|
||||
|
|
||||
|
|
@ -1463,10 +1495,48 @@ Module(
|
|||
|
|
||||
|
||||
|
||||
|
|
||||
15 | with ((*item)): ...
|
||||
16 |
|
||||
17 | with (*x for x in iter, item): ...
|
||||
| ^ Syntax Error: Expected ')', found ','
|
||||
18 | with (item1, *x for x in iter, item2): ...
|
||||
19 | with (x as f, *y): ...
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
15 | with ((*item)): ...
|
||||
16 |
|
||||
17 | with (*x for x in iter, item): ...
|
||||
| ^ Syntax Error: Expected ',', found ')'
|
||||
18 | with (item1, *x for x in iter, item2): ...
|
||||
19 | with (x as f, *y): ...
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
17 | with (*x for x in iter, item): ...
|
||||
18 | with (item1, *x for x in iter, item2): ...
|
||||
| ^^ Syntax Error: Iterable unpacking cannot be used in a comprehension
|
||||
| ^^^ Syntax Error: Expected ',', found 'for'
|
||||
19 | with (x as f, *y): ...
|
||||
20 | with (*x, y as f): ...
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
17 | with (*x for x in iter, item): ...
|
||||
18 | with (item1, *x for x in iter, item2): ...
|
||||
| ^ Syntax Error: Expected ':', found ')'
|
||||
19 | with (x as f, *y): ...
|
||||
20 | with (*x, y as f): ...
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
17 | with (*x for x in iter, item): ...
|
||||
18 | with (item1, *x for x in iter, item2): ...
|
||||
| ^ Syntax Error: Expected a statement
|
||||
19 | with (x as f, *y): ...
|
||||
20 | with (*x, y as f): ...
|
||||
|
|
||||
|
|
@ -1535,7 +1605,17 @@ Module(
|
|||
23 | with (x, yield from y): ...
|
||||
24 | with (x as f, y) as f: ...
|
||||
25 | with (x for x in iter as y): ...
|
||||
| ^^^^^^^^^^^^^^^ Syntax Error: Unparenthesized generator expression cannot be used here
|
||||
| ^^ Syntax Error: Expected ')', found 'as'
|
||||
26 |
|
||||
27 | # The inner `(...)` is parsed as parenthesized expression
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
23 | with (x, yield from y): ...
|
||||
24 | with (x as f, y) as f: ...
|
||||
25 | with (x for x in iter as y): ...
|
||||
| ^ Syntax Error: Expected ',', found ')'
|
||||
26 |
|
||||
27 | # The inner `(...)` is parsed as parenthesized expression
|
||||
|
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ Module(
|
|||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 6..6,
|
||||
range: 5..6,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 6..6,
|
||||
|
|
@ -25,32 +25,34 @@ Module(
|
|||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 9..14,
|
||||
context_expr: BinOp(
|
||||
ExprBinOp {
|
||||
range: 9..14,
|
||||
left: Name(
|
||||
ExprName {
|
||||
range: 9..10,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
op: Add,
|
||||
right: Name(
|
||||
ExprName {
|
||||
range: 13..14,
|
||||
id: "y",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [],
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 9..14,
|
||||
value: BinOp(
|
||||
ExprBinOp {
|
||||
range: 9..14,
|
||||
left: Name(
|
||||
ExprName {
|
||||
range: 9..10,
|
||||
id: "x",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
op: Add,
|
||||
right: Name(
|
||||
ExprName {
|
||||
range: 13..14,
|
||||
id: "y",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ Module(
|
|||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 6..6,
|
||||
range: 5..6,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 6..6,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/tests/fixtures.rs
|
||||
input_file: crates/ruff_python_parser/resources/inline/err/with_items_parenthesized_missing_colon.py
|
||||
---
|
||||
## AST
|
||||
|
||||
```
|
||||
Module(
|
||||
ModModule {
|
||||
range: 0..57,
|
||||
body: [
|
||||
With(
|
||||
StmtWith {
|
||||
range: 28..56,
|
||||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 34..39,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 34..39,
|
||||
id: "item1",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 41..46,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 41..46,
|
||||
id: "item2",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [
|
||||
Pass(
|
||||
StmtPass {
|
||||
range: 52..56,
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
```
|
||||
## Errors
|
||||
|
||||
|
|
||||
1 | # `)` followed by a newline
|
||||
2 | with (item1, item2)
|
||||
| ^ Syntax Error: Expected ':', found newline
|
||||
3 | pass
|
||||
|
|
||||
|
|
@ -239,42 +239,49 @@ Module(
|
|||
),
|
||||
With(
|
||||
StmtWith {
|
||||
range: 136..160,
|
||||
range: 136..159,
|
||||
is_async: false,
|
||||
items: [
|
||||
WithItem {
|
||||
range: 142..147,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 142..147,
|
||||
id: "item1",
|
||||
range: 141..154,
|
||||
context_expr: Tuple(
|
||||
ExprTuple {
|
||||
range: 141..154,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
range: 142..147,
|
||||
id: "item1",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
Name(
|
||||
ExprName {
|
||||
range: 149..154,
|
||||
id: "item2",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
],
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 149..154,
|
||||
context_expr: Name(
|
||||
ExprName {
|
||||
range: 149..154,
|
||||
id: "item2",
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
WithItem {
|
||||
range: 156..159,
|
||||
context_expr: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 156..159,
|
||||
parenthesized: true,
|
||||
},
|
||||
),
|
||||
optional_vars: None,
|
||||
},
|
||||
],
|
||||
body: [],
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 156..159,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 156..159,
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue