[syntax-errors] Named expressions in decorators before Python 3.9 (#16386)

Summary
--

This PR detects the relaxed grammar for decorators proposed in [PEP
614](https://peps.python.org/pep-0614/) on Python 3.8 and lower.

The 3.8 grammar for decorators is
[here](https://docs.python.org/3.8/reference/compound_stmts.html#grammar-token-decorators):

```
decorators                ::=  decorator+
decorator                 ::=  "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE
dotted_name               ::=  identifier ("." identifier)*
```

in contrast to the current grammar
[here](https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-decorators)

```
decorators                ::= decorator+
decorator                 ::= "@" assignment_expression NEWLINE
assignment_expression ::= [identifier ":="] expression
```

Test Plan
--

New inline parser tests.
This commit is contained in:
Brent Westbrook 2025-03-05 12:08:18 -05:00 committed by GitHub
parent d0623888b3
commit 318f503714
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 876 additions and 3 deletions

View file

@ -0,0 +1,3 @@
# parse_options: { "target-version": "3.8" }
@buttons[0].clicked.connect
def spam(): ...

View file

@ -0,0 +1,3 @@
# parse_options: { "target-version": "3.7" }
@(x := lambda x: x)(foo)
def bar(): ...

View file

@ -0,0 +1,3 @@
# parse_options: { "target-version": "3.8" }
@buttons.clicked.connect
def spam(): ...

View file

@ -0,0 +1,3 @@
# parse_options: { "target-version": "3.8" }
@eval("buttons[0].clicked.connect")
def spam(): ...

View file

@ -0,0 +1,4 @@
# parse_options: { "target-version": "3.8" }
def _(x): return x
@_(buttons[0].clicked.connect)
def spam(): ...

View file

@ -0,0 +1,5 @@
# parse_options: { "target-version": "3.9" }
@buttons[0].clicked.connect
def spam(): ...
@(x := lambda x: x)(foo)
def bar(): ...

View file

@ -449,6 +449,37 @@ pub enum UnsupportedSyntaxErrorKind {
Match, Match,
Walrus, Walrus,
ExceptStar, ExceptStar,
/// Represents the use of a "relaxed" [PEP 614] decorator before Python 3.9.
///
/// ## Examples
///
/// Prior to Python 3.9, decorators were defined to be [`dotted_name`]s, optionally followed by
/// an argument list. For example:
///
/// ```python
/// @buttons.clicked.connect
/// def foo(): ...
///
/// @buttons.clicked.connect(1, 2, 3)
/// def foo(): ...
/// ```
///
/// As pointed out in the PEP, this prevented reasonable extensions like subscripts:
///
/// ```python
/// buttons = [QPushButton(f'Button {i}') for i in range(10)]
///
/// @buttons[0].clicked.connect
/// def spam(): ...
/// ```
///
/// Python 3.9 removed these restrictions and expanded the [decorator grammar] to include any
/// assignment expression and include cases like the example above.
///
/// [PEP 614]: https://peps.python.org/pep-0614/
/// [`dotted_name`]: https://docs.python.org/3.8/reference/compound_stmts.html#grammar-token-dotted-name
/// [decorator grammar]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-decorator
RelaxedDecorator,
/// Represents the use of a [PEP 570] positional-only parameter before Python 3.8. /// Represents the use of a [PEP 570] positional-only parameter before Python 3.8.
/// ///
/// ## Examples /// ## Examples
@ -513,6 +544,7 @@ impl Display for UnsupportedSyntaxError {
UnsupportedSyntaxErrorKind::Match => "Cannot use `match` statement", UnsupportedSyntaxErrorKind::Match => "Cannot use `match` statement",
UnsupportedSyntaxErrorKind::Walrus => "Cannot use named assignment expression (`:=`)", UnsupportedSyntaxErrorKind::Walrus => "Cannot use named assignment expression (`:=`)",
UnsupportedSyntaxErrorKind::ExceptStar => "Cannot use `except*`", UnsupportedSyntaxErrorKind::ExceptStar => "Cannot use `except*`",
UnsupportedSyntaxErrorKind::RelaxedDecorator => "Unsupported expression in decorators",
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => { UnsupportedSyntaxErrorKind::PositionalOnlyParameter => {
"Cannot use positional-only parameter separator" "Cannot use positional-only parameter separator"
} }
@ -538,6 +570,7 @@ impl UnsupportedSyntaxErrorKind {
UnsupportedSyntaxErrorKind::Match => PythonVersion::PY310, UnsupportedSyntaxErrorKind::Match => PythonVersion::PY310,
UnsupportedSyntaxErrorKind::Walrus => PythonVersion::PY38, UnsupportedSyntaxErrorKind::Walrus => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::ExceptStar => PythonVersion::PY311, UnsupportedSyntaxErrorKind::ExceptStar => PythonVersion::PY311,
UnsupportedSyntaxErrorKind::RelaxedDecorator => PythonVersion::PY39,
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => PythonVersion::PY38, UnsupportedSyntaxErrorKind::PositionalOnlyParameter => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::TypeParameterList => PythonVersion::PY312, UnsupportedSyntaxErrorKind::TypeParameterList => PythonVersion::PY312,
UnsupportedSyntaxErrorKind::TypeAliasStatement => PythonVersion::PY312, UnsupportedSyntaxErrorKind::TypeAliasStatement => PythonVersion::PY312,

View file

@ -632,7 +632,7 @@ impl<'src> Parser<'src> {
/// If the parser isn't position at a `(` token. /// If the parser isn't position at a `(` token.
/// ///
/// See: <https://docs.python.org/3/reference/expressions.html#calls> /// See: <https://docs.python.org/3/reference/expressions.html#calls>
fn parse_call_expression(&mut self, func: Expr, start: TextSize) -> ast::ExprCall { pub(super) fn parse_call_expression(&mut self, func: Expr, start: TextSize) -> ast::ExprCall {
let arguments = self.parse_arguments(); let arguments = self.parse_arguments();
ast::ExprCall { ast::ExprCall {

View file

@ -43,3 +43,15 @@ pub(super) const fn token_kind_to_cmp_op(tokens: [TokenKind; 2]) -> Option<CmpOp
_ => return None, _ => return None,
}) })
} }
/// Helper for `parse_decorators` to determine if `expr` is a [`dotted_name`] from the decorator
/// grammar before Python 3.9.
///
/// [`dotted_name`]: https://docs.python.org/3.8/reference/compound_stmts.html#grammar-token-dotted-name
pub(super) fn is_name_or_attribute_expression(expr: &Expr) -> bool {
match expr {
Expr::Attribute(attr) => is_name_or_attribute_expression(&attr.value),
Expr::Name(_) => true,
_ => false,
}
}

View file

@ -5,7 +5,8 @@ use rustc_hash::{FxBuildHasher, FxHashSet};
use ruff_python_ast::name::Name; use ruff_python_ast::name::Name;
use ruff_python_ast::{ use ruff_python_ast::{
self as ast, ExceptHandler, Expr, ExprContext, IpyEscapeKind, Operator, Stmt, WithItem, self as ast, ExceptHandler, Expr, ExprContext, IpyEscapeKind, Operator, PythonVersion, Stmt,
WithItem,
}; };
use ruff_text_size::{Ranged, TextRange, TextSize}; use ruff_text_size::{Ranged, TextRange, TextSize};
@ -2599,6 +2600,56 @@ impl<'src> Parser<'src> {
let decorator_start = self.node_start(); let decorator_start = self.node_start();
self.bump(TokenKind::At); self.bump(TokenKind::At);
let parsed_expr = self.parse_named_expression_or_higher(ExpressionContext::default());
if self.options.target_version < PythonVersion::PY39 {
// test_ok decorator_expression_dotted_ident_py38
// # parse_options: { "target-version": "3.8" }
// @buttons.clicked.connect
// def spam(): ...
// test_ok decorator_expression_identity_hack_py38
// # parse_options: { "target-version": "3.8" }
// def _(x): return x
// @_(buttons[0].clicked.connect)
// def spam(): ...
// test_ok decorator_expression_eval_hack_py38
// # parse_options: { "target-version": "3.8" }
// @eval("buttons[0].clicked.connect")
// def spam(): ...
// test_ok decorator_expression_py39
// # parse_options: { "target-version": "3.9" }
// @buttons[0].clicked.connect
// def spam(): ...
// @(x := lambda x: x)(foo)
// def bar(): ...
// test_err decorator_expression_py38
// # parse_options: { "target-version": "3.8" }
// @buttons[0].clicked.connect
// def spam(): ...
// test_err decorator_named_expression_py37
// # parse_options: { "target-version": "3.7" }
// @(x := lambda x: x)(foo)
// def bar(): ...
let allowed_decorator = match &parsed_expr.expr {
Expr::Call(expr_call) => {
helpers::is_name_or_attribute_expression(&expr_call.func)
}
expr => helpers::is_name_or_attribute_expression(expr),
};
if !allowed_decorator {
self.add_unsupported_syntax_error(
UnsupportedSyntaxErrorKind::RelaxedDecorator,
parsed_expr.range(),
);
}
}
// test_err decorator_invalid_expression // test_err decorator_invalid_expression
// @*x // @*x
// @(*x) // @(*x)
@ -2606,7 +2657,6 @@ impl<'src> Parser<'src> {
// @yield x // @yield x
// @yield from x // @yield from x
// def foo(): ... // def foo(): ...
let parsed_expr = self.parse_named_expression_or_higher(ExpressionContext::default());
decorators.push(ast::Decorator { decorators.push(ast::Decorator {
expression: parsed_expr.expr, expression: parsed_expr.expr,

View file

@ -0,0 +1,101 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/decorator_expression_py38.py
---
## AST
```
Module(
ModModule {
range: 0..89,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..88,
is_async: false,
decorator_list: [
Decorator {
range: 45..72,
expression: Attribute(
ExprAttribute {
range: 46..72,
value: Attribute(
ExprAttribute {
range: 46..64,
value: Subscript(
ExprSubscript {
range: 46..56,
value: Name(
ExprName {
range: 46..53,
id: Name("buttons"),
ctx: Load,
},
),
slice: NumberLiteral(
ExprNumberLiteral {
range: 54..55,
value: Int(
0,
),
},
),
ctx: Load,
},
),
attr: Identifier {
id: Name("clicked"),
range: 57..64,
},
ctx: Load,
},
),
attr: Identifier {
id: Name("connect"),
range: 65..72,
},
ctx: Load,
},
),
},
],
name: Identifier {
id: Name("spam"),
range: 77..81,
},
type_params: None,
parameters: Parameters {
range: 81..83,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 85..88,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 85..88,
},
),
},
),
],
},
),
],
},
)
```
## Unsupported Syntax Errors
|
1 | # parse_options: { "target-version": "3.8" }
2 | @buttons[0].clicked.connect
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ Syntax Error: Unsupported expression in decorators on Python 3.8 (syntax was added in Python 3.9)
3 | def spam(): ...
|

View file

@ -0,0 +1,133 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/decorator_named_expression_py37.py
---
## AST
```
Module(
ModModule {
range: 0..85,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..84,
is_async: false,
decorator_list: [
Decorator {
range: 45..69,
expression: Call(
ExprCall {
range: 46..69,
func: Named(
ExprNamed {
range: 47..63,
target: Name(
ExprName {
range: 47..48,
id: Name("x"),
ctx: Store,
},
),
value: Lambda(
ExprLambda {
range: 52..63,
parameters: Some(
Parameters {
range: 59..60,
posonlyargs: [],
args: [
ParameterWithDefault {
range: 59..60,
parameter: Parameter {
range: 59..60,
name: Identifier {
id: Name("x"),
range: 59..60,
},
annotation: None,
},
default: None,
},
],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
),
body: Name(
ExprName {
range: 62..63,
id: Name("x"),
ctx: Load,
},
),
},
),
},
),
arguments: Arguments {
range: 64..69,
args: [
Name(
ExprName {
range: 65..68,
id: Name("foo"),
ctx: Load,
},
),
],
keywords: [],
},
},
),
},
],
name: Identifier {
id: Name("bar"),
range: 74..77,
},
type_params: None,
parameters: Parameters {
range: 77..79,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 81..84,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 81..84,
},
),
},
),
],
},
),
],
},
)
```
## Unsupported Syntax Errors
|
1 | # parse_options: { "target-version": "3.7" }
2 | @(x := lambda x: x)(foo)
| ^^^^^^^^^^^^^^^^ Syntax Error: Cannot use named assignment expression (`:=`) on Python 3.7 (syntax was added in Python 3.8)
3 | def bar(): ...
|
|
1 | # parse_options: { "target-version": "3.7" }
2 | @(x := lambda x: x)(foo)
| ^^^^^^^^^^^^^^^^^^^^^^^ Syntax Error: Unsupported expression in decorators on Python 3.7 (syntax was added in Python 3.9)
3 | def bar(): ...
|

View file

@ -0,0 +1,79 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/decorator_expression_dotted_ident_py38.py
---
## AST
```
Module(
ModModule {
range: 0..86,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..85,
is_async: false,
decorator_list: [
Decorator {
range: 45..69,
expression: Attribute(
ExprAttribute {
range: 46..69,
value: Attribute(
ExprAttribute {
range: 46..61,
value: Name(
ExprName {
range: 46..53,
id: Name("buttons"),
ctx: Load,
},
),
attr: Identifier {
id: Name("clicked"),
range: 54..61,
},
ctx: Load,
},
),
attr: Identifier {
id: Name("connect"),
range: 62..69,
},
ctx: Load,
},
),
},
],
name: Identifier {
id: Name("spam"),
range: 74..78,
},
type_params: None,
parameters: Parameters {
range: 78..80,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 82..85,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 82..85,
},
),
},
),
],
},
),
],
},
)
```

View file

@ -0,0 +1,88 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/decorator_expression_eval_hack_py38.py
---
## AST
```
Module(
ModModule {
range: 0..97,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..96,
is_async: false,
decorator_list: [
Decorator {
range: 45..80,
expression: Call(
ExprCall {
range: 46..80,
func: Name(
ExprName {
range: 46..50,
id: Name("eval"),
ctx: Load,
},
),
arguments: Arguments {
range: 50..80,
args: [
StringLiteral(
ExprStringLiteral {
range: 51..79,
value: StringLiteralValue {
inner: Single(
StringLiteral {
range: 51..79,
value: "buttons[0].clicked.connect",
flags: StringLiteralFlags {
quote_style: Double,
prefix: Empty,
triple_quoted: false,
},
},
),
},
},
),
],
keywords: [],
},
},
),
},
],
name: Identifier {
id: Name("spam"),
range: 85..89,
},
type_params: None,
parameters: Parameters {
range: 89..91,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 93..96,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 93..96,
},
),
},
),
],
},
),
],
},
)
```

View file

@ -0,0 +1,161 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/decorator_expression_identity_hack_py38.py
---
## AST
```
Module(
ModModule {
range: 0..111,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..63,
is_async: false,
decorator_list: [],
name: Identifier {
id: Name("_"),
range: 49..50,
},
type_params: None,
parameters: Parameters {
range: 50..53,
posonlyargs: [],
args: [
ParameterWithDefault {
range: 51..52,
parameter: Parameter {
range: 51..52,
name: Identifier {
id: Name("x"),
range: 51..52,
},
annotation: None,
},
default: None,
},
],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Return(
StmtReturn {
range: 55..63,
value: Some(
Name(
ExprName {
range: 62..63,
id: Name("x"),
ctx: Load,
},
),
),
},
),
],
},
),
FunctionDef(
StmtFunctionDef {
range: 64..110,
is_async: false,
decorator_list: [
Decorator {
range: 64..94,
expression: Call(
ExprCall {
range: 65..94,
func: Name(
ExprName {
range: 65..66,
id: Name("_"),
ctx: Load,
},
),
arguments: Arguments {
range: 66..94,
args: [
Attribute(
ExprAttribute {
range: 67..93,
value: Attribute(
ExprAttribute {
range: 67..85,
value: Subscript(
ExprSubscript {
range: 67..77,
value: Name(
ExprName {
range: 67..74,
id: Name("buttons"),
ctx: Load,
},
),
slice: NumberLiteral(
ExprNumberLiteral {
range: 75..76,
value: Int(
0,
),
},
),
ctx: Load,
},
),
attr: Identifier {
id: Name("clicked"),
range: 78..85,
},
ctx: Load,
},
),
attr: Identifier {
id: Name("connect"),
range: 86..93,
},
ctx: Load,
},
),
],
keywords: [],
},
},
),
},
],
name: Identifier {
id: Name("spam"),
range: 99..103,
},
type_params: None,
parameters: Parameters {
range: 103..105,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 107..110,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 107..110,
},
),
},
),
],
},
),
],
},
)
```

View file

@ -0,0 +1,195 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/ok/decorator_expression_py39.py
---
## AST
```
Module(
ModModule {
range: 0..129,
body: [
FunctionDef(
StmtFunctionDef {
range: 45..88,
is_async: false,
decorator_list: [
Decorator {
range: 45..72,
expression: Attribute(
ExprAttribute {
range: 46..72,
value: Attribute(
ExprAttribute {
range: 46..64,
value: Subscript(
ExprSubscript {
range: 46..56,
value: Name(
ExprName {
range: 46..53,
id: Name("buttons"),
ctx: Load,
},
),
slice: NumberLiteral(
ExprNumberLiteral {
range: 54..55,
value: Int(
0,
),
},
),
ctx: Load,
},
),
attr: Identifier {
id: Name("clicked"),
range: 57..64,
},
ctx: Load,
},
),
attr: Identifier {
id: Name("connect"),
range: 65..72,
},
ctx: Load,
},
),
},
],
name: Identifier {
id: Name("spam"),
range: 77..81,
},
type_params: None,
parameters: Parameters {
range: 81..83,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 85..88,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 85..88,
},
),
},
),
],
},
),
FunctionDef(
StmtFunctionDef {
range: 89..128,
is_async: false,
decorator_list: [
Decorator {
range: 89..113,
expression: Call(
ExprCall {
range: 90..113,
func: Named(
ExprNamed {
range: 91..107,
target: Name(
ExprName {
range: 91..92,
id: Name("x"),
ctx: Store,
},
),
value: Lambda(
ExprLambda {
range: 96..107,
parameters: Some(
Parameters {
range: 103..104,
posonlyargs: [],
args: [
ParameterWithDefault {
range: 103..104,
parameter: Parameter {
range: 103..104,
name: Identifier {
id: Name("x"),
range: 103..104,
},
annotation: None,
},
default: None,
},
],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
),
body: Name(
ExprName {
range: 106..107,
id: Name("x"),
ctx: Load,
},
),
},
),
},
),
arguments: Arguments {
range: 108..113,
args: [
Name(
ExprName {
range: 109..112,
id: Name("foo"),
ctx: Load,
},
),
],
keywords: [],
},
},
),
},
],
name: Identifier {
id: Name("bar"),
range: 118..121,
},
type_params: None,
parameters: Parameters {
range: 121..123,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Expr(
StmtExpr {
range: 125..128,
value: EllipsisLiteral(
ExprEllipsisLiteral {
range: 125..128,
},
),
},
),
],
},
),
],
},
)
```