From 2382fe1f256049615a8914a3778660acc45fc74c Mon Sep 17 00:00:00 2001 From: Brent Westbrook <36778786+ntBre@users.noreply.github.com> Date: Thu, 13 Mar 2025 15:55:17 -0400 Subject: [PATCH] [syntax-errors] Tuple unpacking in `for` statement iterator clause before Python 3.9 (#16558) Summary -- This PR reuses a slightly modified version of the `check_tuple_unpacking` method added for detecting unpacking in `return` and `yield` statements to detect the same issue in the iterator clause of `for` loops. I ran into the same issue with a bare `for x in *rest: ...` example (invalid even on Python 3.13) and added it as a comment on https://github.com/astral-sh/ruff/issues/16520. I considered just making this an additional `StarTupleKind` variant as well, but this change was in a different version of Python, so I kept it separate. Test Plan -- New inline tests. --- .../inline/err/for_iter_unpack_py38.py | 4 + .../inline/ok/for_iter_unpack_py38.py | 4 + .../inline/ok/for_iter_unpack_py39.py | 4 + crates/ruff_python_parser/src/error.rs | 33 +++ .../src/parser/expression.rs | 5 +- .../src/parser/statement.rs | 37 ++- ...nvalid_syntax@for_iter_unpack_py38.py.snap | 220 ++++++++++++++++++ .../valid_syntax@for_iter_unpack_py38.py.snap | 186 +++++++++++++++ .../valid_syntax@for_iter_unpack_py39.py.snap | 186 +++++++++++++++ 9 files changed, 673 insertions(+), 6 deletions(-) create mode 100644 crates/ruff_python_parser/resources/inline/err/for_iter_unpack_py38.py create mode 100644 crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py38.py create mode 100644 crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py39.py create mode 100644 crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_iter_unpack_py38.py.snap create mode 100644 crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py38.py.snap create mode 100644 crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py39.py.snap diff --git a/crates/ruff_python_parser/resources/inline/err/for_iter_unpack_py38.py b/crates/ruff_python_parser/resources/inline/err/for_iter_unpack_py38.py new file mode 100644 index 0000000000..ef95a49cb5 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/for_iter_unpack_py38.py @@ -0,0 +1,4 @@ +# parse_options: {"target-version": "3.8"} +for x in *a, b: ... +for x in a, *b: ... +for x in *a, *b: ... diff --git a/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py38.py b/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py38.py new file mode 100644 index 0000000000..88b3af5c48 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py38.py @@ -0,0 +1,4 @@ +# parse_options: {"target-version": "3.8"} +for x in (*a, b): ... +for x in ( a, *b): ... +for x in (*a, *b): ... diff --git a/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py39.py b/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py39.py new file mode 100644 index 0000000000..1ce6a6b12f --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py39.py @@ -0,0 +1,4 @@ +# parse_options: {"target-version": "3.9"} +for x in *a, b: ... +for x in a, *b: ... +for x in *a, *b: ... diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 6d0f6c3845..2df7ce03a0 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -616,6 +616,33 @@ pub enum UnsupportedSyntaxErrorKind { TypeParameterList, TypeAliasStatement, TypeParamDefault, + + /// Represents the use of tuple unpacking in a `for` statement iterator clause before Python + /// 3.9. + /// + /// ## Examples + /// + /// Like [`UnsupportedSyntaxErrorKind::StarTuple`] in `return` and `yield` statements, prior to + /// Python 3.9, tuple unpacking in the iterator clause of a `for` statement required + /// parentheses: + /// + /// ```python + /// # valid on Python 3.8 and earlier + /// for i in (*a, *b): ... + /// ``` + /// + /// Omitting the parentheses was invalid: + /// + /// ```python + /// for i in *a, *b: ... # SyntaxError + /// ``` + /// + /// This was changed as part of the [PEG parser rewrite] included in Python 3.9 but not + /// documented directly until the [Python 3.11 release]. + /// + /// [PEG parser rewrite]: https://peps.python.org/pep-0617/ + /// [Python 3.11 release]: https://docs.python.org/3/whatsnew/3.11.html#other-language-changes + UnparenthesizedUnpackInFor, } impl Display for UnsupportedSyntaxError { @@ -642,6 +669,9 @@ impl Display for UnsupportedSyntaxError { UnsupportedSyntaxErrorKind::TypeParamDefault => { "Cannot set default type for a type parameter" } + UnsupportedSyntaxErrorKind::UnparenthesizedUnpackInFor => { + "Cannot use iterable unpacking in `for` statements" + } }; write!( @@ -687,6 +717,9 @@ impl UnsupportedSyntaxErrorKind { UnsupportedSyntaxErrorKind::TypeParameterList => Change::Added(PythonVersion::PY312), UnsupportedSyntaxErrorKind::TypeAliasStatement => Change::Added(PythonVersion::PY312), UnsupportedSyntaxErrorKind::TypeParamDefault => Change::Added(PythonVersion::PY313), + UnsupportedSyntaxErrorKind::UnparenthesizedUnpackInFor => { + Change::Added(PythonVersion::PY39) + } } } diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index b5fa489af6..f1e370e68d 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2130,7 +2130,10 @@ impl<'src> Parser<'src> { // rest = (4, 5, 6) // def g(): yield 1, 2, 3, *rest // def h(): yield 1, (yield 2, *rest), 3 - self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Yield); + self.check_tuple_unpacking( + &parsed_expr, + UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Yield), + ); Box::new(parsed_expr.expr) }); diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index f9bde0aa04..e234562c83 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -406,7 +406,10 @@ impl<'src> Parser<'src> { // # parse_options: {"target-version": "3.7"} // rest = (4, 5, 6) // def f(): return 1, 2, 3, *rest - self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Return); + self.check_tuple_unpacking( + &parsed_expr, + UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Return), + ); Box::new(parsed_expr.expr) }); @@ -420,10 +423,12 @@ impl<'src> Parser<'src> { /// Report [`UnsupportedSyntaxError`]s for each starred element in `expr` if it is an /// unparenthesized tuple. /// - /// This method can be used to check for tuple unpacking in return and yield statements, which - /// are only allowed in Python 3.8 and later: . - pub(crate) fn check_tuple_unpacking(&mut self, expr: &Expr, kind: StarTupleKind) { - let kind = UnsupportedSyntaxErrorKind::StarTuple(kind); + /// This method can be used to check for tuple unpacking in `return`, `yield`, and `for` + /// statements, which are only allowed after [Python 3.8] and [Python 3.9], respectively. + /// + /// [Python 3.8]: https://github.com/python/cpython/issues/76298 + /// [Python 3.9]: https://github.com/python/cpython/issues/90881 + pub(super) fn check_tuple_unpacking(&mut self, expr: &Expr, kind: UnsupportedSyntaxErrorKind) { if kind.is_supported(self.options.target_version) { return; } @@ -1732,6 +1737,28 @@ impl<'src> Parser<'src> { // for target in x := 1: ... let iter = self.parse_expression_list(ExpressionContext::starred_bitwise_or()); + // test_ok for_iter_unpack_py39 + // # parse_options: {"target-version": "3.9"} + // for x in *a, b: ... + // for x in a, *b: ... + // for x in *a, *b: ... + + // test_ok for_iter_unpack_py38 + // # parse_options: {"target-version": "3.8"} + // for x in (*a, b): ... + // for x in ( a, *b): ... + // for x in (*a, *b): ... + + // test_err for_iter_unpack_py38 + // # parse_options: {"target-version": "3.8"} + // for x in *a, b: ... + // for x in a, *b: ... + // for x in *a, *b: ... + self.check_tuple_unpacking( + &iter, + UnsupportedSyntaxErrorKind::UnparenthesizedUnpackInFor, + ); + self.expect(TokenKind::Colon); let body = self.parse_body(Clause::For); diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_iter_unpack_py38.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_iter_unpack_py38.py.snap new file mode 100644 index 0000000000..f4a68c5000 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_iter_unpack_py38.py.snap @@ -0,0 +1,220 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/for_iter_unpack_py38.py +--- +## AST + +``` +Module( + ModModule { + range: 0..106, + body: [ + For( + StmtFor { + range: 43..63, + is_async: false, + target: Name( + ExprName { + range: 47..48, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 52..58, + elts: [ + Starred( + ExprStarred { + range: 52..54, + value: Name( + ExprName { + range: 53..54, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Name( + ExprName { + range: 57..58, + id: Name("b"), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 60..63, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 60..63, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 64..84, + is_async: false, + target: Name( + ExprName { + range: 68..69, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 74..79, + elts: [ + Name( + ExprName { + range: 74..75, + id: Name("a"), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 77..79, + value: Name( + ExprName { + range: 78..79, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 81..84, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 81..84, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 85..105, + is_async: false, + target: Name( + ExprName { + range: 89..90, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 94..100, + elts: [ + Starred( + ExprStarred { + range: 94..96, + value: Name( + ExprName { + range: 95..96, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 98..100, + value: Name( + ExprName { + range: 99..100, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 102..105, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 102..105, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + ], + }, +) +``` +## Unsupported Syntax Errors + + | +1 | # parse_options: {"target-version": "3.8"} +2 | for x in *a, b: ... + | ^^ Syntax Error: Cannot use iterable unpacking in `for` statements on Python 3.8 (syntax was added in Python 3.9) +3 | for x in a, *b: ... +4 | for x in *a, *b: ... + | + + + | +1 | # parse_options: {"target-version": "3.8"} +2 | for x in *a, b: ... +3 | for x in a, *b: ... + | ^^ Syntax Error: Cannot use iterable unpacking in `for` statements on Python 3.8 (syntax was added in Python 3.9) +4 | for x in *a, *b: ... + | + + + | +2 | for x in *a, b: ... +3 | for x in a, *b: ... +4 | for x in *a, *b: ... + | ^^ Syntax Error: Cannot use iterable unpacking in `for` statements on Python 3.8 (syntax was added in Python 3.9) + | + + + | +2 | for x in *a, b: ... +3 | for x in a, *b: ... +4 | for x in *a, *b: ... + | ^^ Syntax Error: Cannot use iterable unpacking in `for` statements on Python 3.8 (syntax was added in Python 3.9) + | diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py38.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py38.py.snap new file mode 100644 index 0000000000..892b22f86a --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py38.py.snap @@ -0,0 +1,186 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py38.py +--- +## AST + +``` +Module( + ModModule { + range: 0..112, + body: [ + For( + StmtFor { + range: 43..65, + is_async: false, + target: Name( + ExprName { + range: 47..48, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 52..60, + elts: [ + Starred( + ExprStarred { + range: 53..55, + value: Name( + ExprName { + range: 54..55, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Name( + ExprName { + range: 58..59, + id: Name("b"), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + body: [ + Expr( + StmtExpr { + range: 62..65, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 62..65, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 66..88, + is_async: false, + target: Name( + ExprName { + range: 70..71, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 75..83, + elts: [ + Name( + ExprName { + range: 77..78, + id: Name("a"), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 80..82, + value: Name( + ExprName { + range: 81..82, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + body: [ + Expr( + StmtExpr { + range: 85..88, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 85..88, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 89..111, + is_async: false, + target: Name( + ExprName { + range: 93..94, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 98..106, + elts: [ + Starred( + ExprStarred { + range: 99..101, + value: Name( + ExprName { + range: 100..101, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 103..105, + value: Name( + ExprName { + range: 104..105, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + body: [ + Expr( + StmtExpr { + range: 108..111, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 108..111, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + ], + }, +) +``` diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py39.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py39.py.snap new file mode 100644 index 0000000000..ac67500fd7 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_iter_unpack_py39.py.snap @@ -0,0 +1,186 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/for_iter_unpack_py39.py +--- +## AST + +``` +Module( + ModModule { + range: 0..106, + body: [ + For( + StmtFor { + range: 43..63, + is_async: false, + target: Name( + ExprName { + range: 47..48, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 52..58, + elts: [ + Starred( + ExprStarred { + range: 52..54, + value: Name( + ExprName { + range: 53..54, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Name( + ExprName { + range: 57..58, + id: Name("b"), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 60..63, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 60..63, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 64..84, + is_async: false, + target: Name( + ExprName { + range: 68..69, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 74..79, + elts: [ + Name( + ExprName { + range: 74..75, + id: Name("a"), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 77..79, + value: Name( + ExprName { + range: 78..79, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 81..84, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 81..84, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 85..105, + is_async: false, + target: Name( + ExprName { + range: 89..90, + id: Name("x"), + ctx: Store, + }, + ), + iter: Tuple( + ExprTuple { + range: 94..100, + elts: [ + Starred( + ExprStarred { + range: 94..96, + value: Name( + ExprName { + range: 95..96, + id: Name("a"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + Starred( + ExprStarred { + range: 98..100, + value: Name( + ExprName { + range: 99..100, + id: Name("b"), + ctx: Load, + }, + ), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: false, + }, + ), + body: [ + Expr( + StmtExpr { + range: 102..105, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 102..105, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + ], + }, +) +```