mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-31 23:57:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add parsing of type alias statements i.e. the type keyword (#97)

Browse source 
Extends #95
Closes #82 

Adds parsing of new `type` soft keyword for defining type aliases.

Supports type alias statements as defined in PEP 695 e.g. 

```python
# A non-generic type alias
type IntOrStr = int | str

# A generic type alias
type ListOrSet[T] = list[T] | set[T]

# A type alias that includes a forward reference
type AnimalOrVegetable = Animal | "Vegetable"

# A generic self-referential type alias
type RecursiveList[T] = T | list[RecursiveList[T]]
```

All type parameter kinds are supported as in #95.

Builds on soft keyword abstractions introduced in https://github.com/RustPython/RustPython/pull/4519
This commit is contained in:
Zanie Blue 2023-07-17 10:00:08 -05:00 • committed by GitHub
parent 6980037ad9
commit 704eb40108
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 20667 additions and 18194 deletions
Download patch file Download diff file Expand all files Collapse all files

1
parser/build.rs
View file

@ -154,6 +154,7 @@ fn gen_phf(out_dir: &Path) {
.entry("raise", "Tok::Raise") .entry("raise", "Tok::Raise")
.entry("return", "Tok::Return") .entry("return", "Tok::Return")
.entry("try", "Tok::Try") .entry("try", "Tok::Try")
.entry("type", "Tok::Type")
.entry("while", "Tok::While") .entry("while", "Tok::While")
.entry("with", "Tok::With") .entry("with", "Tok::With")
.entry("yield", "Tok::Yield") .entry("yield", "Tok::Yield")

92
parser/src/parser.rs
View file

@ -894,11 +894,92 @@ except* OSError as e:
assert!(parse(source, Mode::Interactive, "<embedded>").is_ok()); assert!(parse(source, Mode::Interactive, "<embedded>").is_ok());
} }
#[test]
#[cfg(feature = "all-nodes-with-ranges")]
fn test_parse_type_declaration() {
let source = r#"
type X = int
type X = int | str
type X = int | "ForwardRefY"
type X[T] = T | list[X[T]] # recursive
type X[T] = int
type X[T] = list[T] | set[T]
type X[T, *Ts, **P] = (T, Ts, P)
type X[T: int, *Ts, **P] = (T, Ts, P)
type X[T: (int, str), *Ts, **P] = (T, Ts, P)
# soft keyword as alias name
type type = int
type match = int
type case = int
# soft keyword as value
type foo = type
type foo = match
type foo = case
# multine definitions
type \
X = int
type X \
= int
type X = \
int
type X = (
int
)
type \
X[T] = T
type X \
[T] = T
type X[T] \
= T
"#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
#[cfg(feature = "all-nodes-with-ranges")]
fn test_type_as_identifier() {
let source = r#"\
type *a + b, c # ((type * a) + b), c
type *(a + b), c # (type * (a + b)), c
type (*a + b, c) # type ((*(a + b)), c)
type -a * b + c # (type - (a * b)) + c
type -(a * b) + c # (type - (a * b)) + c
type (-a) * b + c # (type (-(a * b))) + c
type ().a # (type()).a
type (()).a # (type(())).a
type ((),).a # (type(())).a
type [a].b # (type[a]).b
type [a,].b # (type[(a,)]).b (not (type[a]).b)
type [(a,)].b # (type[(a,)]).b
type()[a:
b] # (type())[a: b]
if type := 1: pass
type = lambda query: query == event
print(type(12))
type(type)
a = (
type in C
)
a = (
type(b)
)
type (
X = int
)
type = 1
type = x = 1
x = type = 1
"#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test] #[test]
#[cfg(feature = "all-nodes-with-ranges")] #[cfg(feature = "all-nodes-with-ranges")]
fn test_match_as_identifier() { fn test_match_as_identifier() {
let parse_ast = ast::Suite::parse( let source = r#"\
r#"
match *a + b, c # ((match * a) + b), c match *a + b, c # ((match * a) + b), c
match *(a + b), c # (match * (a + b)), c match *(a + b), c # (match * (a + b)), c
match (*a + b, c) # match ((*(a + b)), c) match (*a + b, c) # match ((*(a + b)), c)
@ -920,11 +1001,8 @@ match match:
pass pass
match = lambda query: query == event match = lambda query: query == event
print(match(12)) print(match(12))
"#, "#;
"<test>", insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]

21
parser/src/python.lalrpop
View file

@ -86,6 +86,7 @@ SmallStatement: ast::Stmt = {
GlobalStatement, GlobalStatement,
NonlocalStatement, NonlocalStatement,
AssertStatement, AssertStatement,
TypeAliasStatement,
}; };
PassStatement: ast::Stmt = { PassStatement: ast::Stmt = {
@ -978,6 +979,25 @@ FuncDef: ast::Stmt = {
}, },
}; };
TypeAliasName: ast::Expr = {
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::Name(
ast::ExprName { id: name, ctx: ast::ExprContext::Load, range: (location..end_location).into() },
),
}
TypeAliasStatement: ast::Stmt = {
<location:@L> "type" <name:TypeAliasName> <type_params:TypeParamList?> "=" <value:Test<"all">> <end_location:@R> => {
ast::Stmt::TypeAlias(
ast::StmtTypeAlias {
name: Box::new(name),
value: Box::new(value),
type_params: type_params.unwrap_or_default(),
range: (location..end_location).into()
},
)
},
};
Parameters: ast::Arguments = { Parameters: ast::Arguments = {
<location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter, DoubleStarTypedParameter>)?> ")" <end_location:@R> =>? { <location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter, DoubleStarTypedParameter>)?> ")" <end_location:@R> =>? {
a.as_ref().map(validate_arguments).transpose()?; a.as_ref().map(validate_arguments).transpose()?;
@ -1750,6 +1770,7 @@ extern {
"raise" => token::Tok::Raise, "raise" => token::Tok::Raise,
"return" => token::Tok::Return, "return" => token::Tok::Return,
"try" => token::Tok::Try, "try" => token::Tok::Try,
"type" => token::Tok::Type,
"while" => token::Tok::While, "while" => token::Tok::While,
"match" => token::Tok::Match, "match" => token::Tok::Match,
"case" => token::Tok::Case, "case" => token::Tok::Case,

36405
parser/src/python.rs generated
View file

File diff suppressed because it is too large Load diff

236
parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap generated
View file

@ -1,24 +1,24 @@
--- ---
source: parser/src/parser.rs source: parser/src/parser.rs
expression: parse_ast expression: "ast::Suite::parse(source, \"<test>\").unwrap()"
--- ---
[ [
Expr( Expr(
StmtExpr { StmtExpr {
range: 1..16, range: 2..17,
value: Tuple( value: Tuple(
ExprTuple { ExprTuple {
range: 1..16, range: 2..17,
elts: [ elts: [
BinOp( BinOp(
ExprBinOp { ExprBinOp {
range: 1..13, range: 2..14,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 1..9, range: 2..10,
left: Name( left: Name(
ExprName { ExprName {
range: 1..6, range: 2..7,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -28,7 +28,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 8..9, range: 9..10,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -40,7 +40,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 12..13, range: 13..14,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -51,7 +51,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 15..16, range: 16..17,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -66,17 +66,17 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 42..59, range: 43..60,
value: Tuple( value: Tuple(
ExprTuple { ExprTuple {
range: 42..59, range: 43..60,
elts: [ elts: [
BinOp( BinOp(
ExprBinOp { ExprBinOp {
range: 42..56, range: 43..57,
left: Name( left: Name(
ExprName { ExprName {
range: 42..47, range: 43..48,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -86,10 +86,10 @@ expression: parse_ast
op: Mult, op: Mult,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 50..55, range: 51..56,
left: Name( left: Name(
ExprName { ExprName {
range: 50..51, range: 51..52,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -99,7 +99,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 54..55, range: 55..56,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -112,7 +112,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 58..59, range: 59..60,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -127,13 +127,13 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 85..102, range: 86..103,
value: Call( value: Call(
ExprCall { ExprCall {
range: 85..102, range: 86..103,
func: Name( func: Name(
ExprName { ExprName {
range: 85..90, range: 86..91,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -143,13 +143,13 @@ expression: parse_ast
args: [ args: [
Starred( Starred(
ExprStarred { ExprStarred {
range: 92..98, range: 93..99,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 93..98, range: 94..99,
left: Name( left: Name(
ExprName { ExprName {
range: 93..94, range: 94..95,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -159,7 +159,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 97..98, range: 98..99,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -173,7 +173,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 100..101, range: 101..102,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -188,16 +188,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 129..145, range: 130..146,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 129..145, range: 130..146,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 129..141, range: 130..142,
left: Name( left: Name(
ExprName { ExprName {
range: 129..134, range: 130..135,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -207,10 +207,10 @@ expression: parse_ast
op: Sub, op: Sub,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 136..141, range: 137..142,
left: Name( left: Name(
ExprName { ExprName {
range: 136..137, range: 137..138,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -220,7 +220,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 140..141, range: 141..142,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -234,7 +234,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 144..145, range: 145..146,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -247,16 +247,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 172..190, range: 173..191,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 172..190, range: 173..191,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 172..186, range: 173..187,
left: Name( left: Name(
ExprName { ExprName {
range: 172..177, range: 173..178,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -266,10 +266,10 @@ expression: parse_ast
op: Sub, op: Sub,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 180..185, range: 181..186,
left: Name( left: Name(
ExprName { ExprName {
range: 180..181, range: 181..182,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -279,7 +279,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 184..185, range: 185..186,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -293,7 +293,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 189..190, range: 190..191,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -306,19 +306,19 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 217..235, range: 218..236,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 217..235, range: 218..236,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 217..231, range: 218..232,
left: Call( left: Call(
ExprCall { ExprCall {
range: 217..227, range: 218..228,
func: Name( func: Name(
ExprName { ExprName {
range: 217..222, range: 218..223,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -328,11 +328,11 @@ expression: parse_ast
args: [ args: [
UnaryOp( UnaryOp(
ExprUnaryOp { ExprUnaryOp {
range: 224..226, range: 225..227,
op: USub, op: USub,
operand: Name( operand: Name(
ExprName { ExprName {
range: 225..226, range: 226..227,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -348,7 +348,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 230..231, range: 231..232,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -360,7 +360,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 234..235, range: 235..236,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -373,16 +373,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 263..273, range: 264..274,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 263..273, range: 264..274,
value: Call( value: Call(
ExprCall { ExprCall {
range: 263..271, range: 264..272,
func: Name( func: Name(
ExprName { ExprName {
range: 263..268, range: 264..269,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -403,16 +403,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 290..302, range: 291..303,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 290..302, range: 291..303,
value: Call( value: Call(
ExprCall { ExprCall {
range: 290..300, range: 291..301,
func: Name( func: Name(
ExprName { ExprName {
range: 290..295, range: 291..296,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -422,7 +422,7 @@ expression: parse_ast
args: [ args: [
Tuple( Tuple(
ExprTuple { ExprTuple {
range: 297..299, range: 298..300,
elts: [], elts: [],
ctx: Load, ctx: Load,
}, },
@ -441,16 +441,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 321..334, range: 322..335,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 321..334, range: 322..335,
value: Call( value: Call(
ExprCall { ExprCall {
range: 321..332, range: 322..333,
func: Name( func: Name(
ExprName { ExprName {
range: 321..326, range: 322..327,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -460,7 +460,7 @@ expression: parse_ast
args: [ args: [
Tuple( Tuple(
ExprTuple { ExprTuple {
range: 328..330, range: 329..331,
elts: [], elts: [],
ctx: Load, ctx: Load,
}, },
@ -479,16 +479,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 353..364, range: 354..365,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 353..364, range: 354..365,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 353..362, range: 354..363,
value: Name( value: Name(
ExprName { ExprName {
range: 353..358, range: 354..359,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -497,7 +497,7 @@ expression: parse_ast
), ),
slice: Name( slice: Name(
ExprName { ExprName {
range: 360..361, range: 361..362,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -517,16 +517,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 382..394, range: 383..395,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 382..394, range: 383..395,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 382..392, range: 383..393,
value: Name( value: Name(
ExprName { ExprName {
range: 382..387, range: 383..388,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -535,11 +535,11 @@ expression: parse_ast
), ),
slice: Tuple( slice: Tuple(
ExprTuple { ExprTuple {
range: 389..391, range: 390..392,
elts: [ elts: [
Name( Name(
ExprName { ExprName {
range: 389..390, range: 390..391,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -563,16 +563,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 435..449, range: 436..450,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 435..449, range: 436..450,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 435..447, range: 436..448,
value: Name( value: Name(
ExprName { ExprName {
range: 435..440, range: 436..441,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -581,11 +581,11 @@ expression: parse_ast
), ),
slice: Tuple( slice: Tuple(
ExprTuple { ExprTuple {
range: 442..446, range: 443..447,
elts: [ elts: [
Name( Name(
ExprName { ExprName {
range: 443..444, range: 444..445,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -609,16 +609,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 470..487, range: 471..488,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 470..487, range: 471..488,
value: Call( value: Call(
ExprCall { ExprCall {
range: 470..477, range: 471..478,
func: Name( func: Name(
ExprName { ExprName {
range: 470..475, range: 471..476,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -631,11 +631,11 @@ expression: parse_ast
), ),
slice: Slice( slice: Slice(
ExprSlice { ExprSlice {
range: 478..486, range: 479..487,
lower: Some( lower: Some(
Name( Name(
ExprName { ExprName {
range: 478..479, range: 479..480,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -646,7 +646,7 @@ expression: parse_ast
upper: Some( upper: Some(
Name( Name(
ExprName { ExprName {
range: 485..486, range: 486..487,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -664,13 +664,13 @@ expression: parse_ast
), ),
If( If(
StmtIf { StmtIf {
range: 507..526, range: 508..527,
test: NamedExpr( test: NamedExpr(
ExprNamedExpr { ExprNamedExpr {
range: 510..520, range: 511..521,
target: Name( target: Name(
ExprName { ExprName {
range: 510..515, range: 511..516,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -679,7 +679,7 @@ expression: parse_ast
), ),
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 519..520, range: 520..521,
value: Int( value: Int(
1, 1,
), ),
@ -691,7 +691,7 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 522..526, range: 523..527,
}, },
), ),
], ],
@ -700,10 +700,10 @@ expression: parse_ast
), ),
Match( Match(
StmtMatch { StmtMatch {
range: 527..581, range: 528..582,
subject: Name( subject: Name(
ExprName { ExprName {
range: 533..538, range: 534..539,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -712,13 +712,13 @@ expression: parse_ast
), ),
cases: [ cases: [
MatchCase { MatchCase {
range: 544..556, range: 545..557,
pattern: MatchValue( pattern: MatchValue(
PatternMatchValue { PatternMatchValue {
range: 549..550, range: 550..551,
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 549..550, range: 550..551,
value: Int( value: Int(
1, 1,
), ),
@ -731,19 +731,19 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 552..556, range: 553..557,
}, },
), ),
], ],
}, },
MatchCase { MatchCase {
range: 561..581, range: 562..582,
pattern: MatchValue( pattern: MatchValue(
PatternMatchValue { PatternMatchValue {
range: 566..567, range: 567..568,
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 566..567, range: 567..568,
value: Int( value: Int(
2, 2,
), ),
@ -756,7 +756,7 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 577..581, range: 578..582,
}, },
), ),
], ],
@ -766,11 +766,11 @@ expression: parse_ast
), ),
Assign( Assign(
StmtAssign { StmtAssign {
range: 582..618, range: 583..619,
targets: [ targets: [
Name( Name(
ExprName { ExprName {
range: 582..587, range: 583..588,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -780,15 +780,15 @@ expression: parse_ast
], ],
value: Lambda( value: Lambda(
ExprLambda { ExprLambda {
range: 590..618, range: 591..619,
args: Arguments { args: Arguments {
range: 597..602, range: 598..603,
posonlyargs: [], posonlyargs: [],
args: [ args: [
ArgWithDefault { ArgWithDefault {
range: 597..602, range: 598..603,
def: Arg { def: Arg {
range: 597..602, range: 598..603,
arg: Identifier( arg: Identifier(
"query", "query",
), ),
@ -804,10 +804,10 @@ expression: parse_ast
}, },
body: Compare( body: Compare(
ExprCompare { ExprCompare {
range: 604..618, range: 605..619,
left: Name( left: Name(
ExprName { ExprName {
range: 604..609, range: 605..610,
id: Identifier( id: Identifier(
"query", "query",
), ),
@ -820,7 +820,7 @@ expression: parse_ast
comparators: [ comparators: [
Name( Name(
ExprName { ExprName {
range: 613..618, range: 614..619,
id: Identifier( id: Identifier(
"event", "event",
), ),
@ -837,13 +837,13 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 619..635, range: 620..636,
value: Call( value: Call(
ExprCall { ExprCall {
range: 619..635, range: 620..636,
func: Name( func: Name(
ExprName { ExprName {
range: 619..624, range: 620..625,
id: Identifier( id: Identifier(
"print", "print",
), ),
@ -853,10 +853,10 @@ expression: parse_ast
args: [ args: [
Call( Call(
ExprCall { ExprCall {
range: 625..634, range: 626..635,
func: Name( func: Name(
ExprName { ExprName {
range: 625..630, range: 626..631,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -866,7 +866,7 @@ expression: parse_ast
args: [ args: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 631..633, range: 632..634,
value: Int( value: Int(
12, 12,
), ),

907
parser/src/snapshots/rustpython_parser__parser__tests__parse_type_declaration.snap generated Normal file
View file

@ -0,0 +1,907 @@
---
source: parser/src/parser.rs
expression: "ast::Suite::parse(source, \"<test>\").unwrap()"
---
[
TypeAlias(
StmtTypeAlias {
range: 1..13,
name: Name(
ExprName {
range: 6..7,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 10..13,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 14..32,
name: Name(
ExprName {
range: 19..20,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: BinOp(
ExprBinOp {
range: 23..32,
left: Name(
ExprName {
range: 23..26,
id: Identifier(
"int",
),
ctx: Load,
},
),
op: BitOr,
right: Name(
ExprName {
range: 29..32,
id: Identifier(
"str",
),
ctx: Load,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 33..61,
name: Name(
ExprName {
range: 38..39,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: BinOp(
ExprBinOp {
range: 42..61,
left: Name(
ExprName {
range: 42..45,
id: Identifier(
"int",
),
ctx: Load,
},
),
op: BitOr,
right: Constant(
ExprConstant {
range: 48..61,
value: Str(
"ForwardRefY",
),
kind: None,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 62..88,
name: Name(
ExprName {
range: 67..68,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 69..70,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: BinOp(
ExprBinOp {
range: 74..88,
left: Name(
ExprName {
range: 74..75,
id: Identifier(
"T",
),
ctx: Load,
},
),
op: BitOr,
right: Subscript(
ExprSubscript {
range: 78..88,
value: Name(
ExprName {
range: 78..82,
id: Identifier(
"list",
),
ctx: Load,
},
),
slice: Subscript(
ExprSubscript {
range: 83..87,
value: Name(
ExprName {
range: 83..84,
id: Identifier(
"X",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 85..86,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
ctx: Load,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 102..117,
name: Name(
ExprName {
range: 107..108,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 109..110,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: Name(
ExprName {
range: 114..117,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 118..146,
name: Name(
ExprName {
range: 123..124,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 125..126,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: BinOp(
ExprBinOp {
range: 130..146,
left: Subscript(
ExprSubscript {
range: 130..137,
value: Name(
ExprName {
range: 130..134,
id: Identifier(
"list",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 135..136,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
op: BitOr,
right: Subscript(
ExprSubscript {
range: 140..146,
value: Name(
ExprName {
range: 140..143,
id: Identifier(
"set",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 144..145,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 147..179,
name: Name(
ExprName {
range: 152..153,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 154..155,
name: Identifier(
"T",
),
bound: None,
},
),
TypeVarTuple(
TypeParamTypeVarTuple {
range: 157..160,
name: Identifier(
"Ts",
),
},
),
ParamSpec(
TypeParamParamSpec {
range: 162..165,
name: Identifier(
"P",
),
},
),
],
value: Tuple(
ExprTuple {
range: 169..179,
elts: [
Name(
ExprName {
range: 170..171,
id: Identifier(
"T",
),
ctx: Load,
},
),
Name(
ExprName {
range: 173..175,
id: Identifier(
"Ts",
),
ctx: Load,
},
),
Name(
ExprName {
range: 177..178,
id: Identifier(
"P",
),
ctx: Load,
},
),
],
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 180..217,
name: Name(
ExprName {
range: 185..186,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 187..193,
name: Identifier(
"T",
),
bound: Some(
Name(
ExprName {
range: 190..193,
id: Identifier(
"int",
),
ctx: Load,
},
),
),
},
),
TypeVarTuple(
TypeParamTypeVarTuple {
range: 195..198,
name: Identifier(
"Ts",
),
},
),
ParamSpec(
TypeParamParamSpec {
range: 200..203,
name: Identifier(
"P",
),
},
),
],
value: Tuple(
ExprTuple {
range: 207..217,
elts: [
Name(
ExprName {
range: 208..209,
id: Identifier(
"T",
),
ctx: Load,
},
),
Name(
ExprName {
range: 211..213,
id: Identifier(
"Ts",
),
ctx: Load,
},
),
Name(
ExprName {
range: 215..216,
id: Identifier(
"P",
),
ctx: Load,
},
),
],
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 218..262,
name: Name(
ExprName {
range: 223..224,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 225..238,
name: Identifier(
"T",
),
bound: Some(
Tuple(
ExprTuple {
range: 228..238,
elts: [
Name(
ExprName {
range: 229..232,
id: Identifier(
"int",
),
ctx: Load,
},
),
Name(
ExprName {
range: 234..237,
id: Identifier(
"str",
),
ctx: Load,
},
),
],
ctx: Load,
},
),
),
},
),
TypeVarTuple(
TypeParamTypeVarTuple {
range: 240..243,
name: Identifier(
"Ts",
),
},
),
ParamSpec(
TypeParamParamSpec {
range: 245..248,
name: Identifier(
"P",
),
},
),
],
value: Tuple(
ExprTuple {
range: 252..262,
elts: [
Name(
ExprName {
range: 253..254,
id: Identifier(
"T",
),
ctx: Load,
},
),
Name(
ExprName {
range: 256..258,
id: Identifier(
"Ts",
),
ctx: Load,
},
),
Name(
ExprName {
range: 260..261,
id: Identifier(
"P",
),
ctx: Load,
},
),
],
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 293..308,
name: Name(
ExprName {
range: 298..302,
id: Identifier(
"type",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 305..308,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 311..327,
name: Name(
ExprName {
range: 316..321,
id: Identifier(
"match",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 324..327,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 328..343,
name: Name(
ExprName {
range: 333..337,
id: Identifier(
"case",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 340..343,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 369..384,
name: Name(
ExprName {
range: 374..377,
id: Identifier(
"foo",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 380..384,
id: Identifier(
"type",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 385..401,
name: Name(
ExprName {
range: 390..393,
id: Identifier(
"foo",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 396..401,
id: Identifier(
"match",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 402..417,
name: Name(
ExprName {
range: 407..410,
id: Identifier(
"foo",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 413..417,
id: Identifier(
"case",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 441..456,
name: Name(
ExprName {
range: 449..450,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 453..456,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 457..472,
name: Name(
ExprName {
range: 462..463,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 469..472,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 473..488,
name: Name(
ExprName {
range: 478..479,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 485..488,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 489..509,
name: Name(
ExprName {
range: 494..495,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [],
value: Name(
ExprName {
range: 504..507,
id: Identifier(
"int",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 510..529,
name: Name(
ExprName {
range: 521..522,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 523..524,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: Name(
ExprName {
range: 528..529,
id: Identifier(
"T",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 530..550,
name: Name(
ExprName {
range: 535..536,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 544..545,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: Name(
ExprName {
range: 549..550,
id: Identifier(
"T",
),
ctx: Load,
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 551..570,
name: Name(
ExprName {
range: 556..557,
id: Identifier(
"X",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 558..559,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: Name(
ExprName {
range: 569..570,
id: Identifier(
"T",
),
ctx: Load,
},
),
},
),
]

1074
parser/src/snapshots/rustpython_parser__parser__tests__type_as_identifier.snap generated Normal file
View file

File diff suppressed because it is too large Load diff

123
parser/src/soft_keywords.rs
View file

@ -2,14 +2,17 @@ use crate::{lexer::LexResult, token::Tok, Mode};
use itertools::{Itertools, MultiPeek}; use itertools::{Itertools, MultiPeek};
/// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match` /// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match`
/// and `case`). /// `case`, and `type`).
/// ///
/// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords /// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords
/// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain /// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain
/// contexts. /// contexts.
/// ///
/// Later, [PEP 695](https://peps.python.org/pep-0695/#generic-type-alias) introduced the `type`
/// soft keyword.
///
/// This function modifies a token stream to accommodate this change. In particular, it replaces /// This function modifies a token stream to accommodate this change. In particular, it replaces
/// `match` and `case` tokens with `identifier` tokens if they are used as identifiers. /// soft keyword tokens with `identifier` tokens if they are used as identifiers.
/// ///
/// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and /// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
/// parser, as neither of them need to be aware of soft keywords. /// parser, as neither of them need to be aware of soft keywords.
@ -43,44 +46,89 @@ where
fn next(&mut self) -> Option<LexResult> { fn next(&mut self) -> Option<LexResult> {
let mut next = self.underlying.next(); let mut next = self.underlying.next();
if let Some(Ok((tok, range))) = next.as_ref() { if let Some(Ok((tok, range))) = next.as_ref() {
// If the token is a `match` or `case` token, check if it's used as an identifier. // If the token is a soft keyword e.g. `type`, `match`, or `case`, check if it's
// We assume every `match` or `case` is an identifier unless both of the following // used as an identifier. We assume every soft keyword use is an identifier unless
// conditions are met: // a heuristic is met.
// 1. The token is at the start of a logical line.
// 2. The logical line contains a top-level colon (that is, a colon that is not nested match tok {
// inside a parenthesized expression, list, or dictionary). // For `match` and `case`, all of the following conditions must be met:
// 3. The top-level colon is not the immediate sibling of a `match` or `case` token. // 1. The token is at the start of a logical line.
// (This is to avoid treating `match` and `case` as identifiers when annotated with // 2. The logical line contains a top-level colon (that is, a colon that is not nested
// type hints.) // inside a parenthesized expression, list, or dictionary).
if matches!(tok, Tok::Match | Tok::Case) { // 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
if !self.start_of_line { // (This is to avoid treating `match` or `case` as identifiers when annotated with
next = Some(Ok((soft_to_name(tok), *range))); // type hints.) type hints.)
} else { Tok::Match | Tok::Case => {
let mut nesting = 0; if !self.start_of_line {
let mut first = true;
let mut seen_colon = false;
let mut seen_lambda = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Lambda if nesting == 0 => seen_lambda = true,
Tok::Colon if nesting == 0 => {
if seen_lambda {
seen_lambda = false;
} else if !first {
seen_colon = true;
}
}
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
}
first = false;
}
if !seen_colon {
next = Some(Ok((soft_to_name(tok), *range))); next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0;
let mut first = true;
let mut seen_colon = false;
let mut seen_lambda = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Lambda if nesting == 0 => seen_lambda = true,
Tok::Colon if nesting == 0 => {
if seen_lambda {
seen_lambda = false;
} else if !first {
seen_colon = true;
}
}
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
}
first = false;
}
if !seen_colon {
next = Some(Ok((soft_to_name(tok), *range)));
}
} }
} }
// For `type` all of the following conditions must be met:
// 1. The token is at the start of a logical line.
// 2. The type token is immediately followed by a name token.
// 3. The name token is eventually followed by an equality token.
Tok::Type => {
if !self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut is_type_alias = false;
if let Some(Ok((tok, _))) = self.underlying.peek() {
if matches!(
tok,
Tok::Name { .. } |
// We treat a soft keyword token following a type token as a
// name to support cases like `type type = int` or `type match = int`
Tok::Type | Tok::Match | Tok::Case
) {
let mut nesting = 0;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Equal if nesting == 0 => {
is_type_alias = true;
break;
}
Tok::Lsqb => nesting += 1,
Tok::Rsqb => nesting -= 1,
// Allow arbitrary content within brackets for now
_ if nesting > 0 => {}
// Exit if unexpected tokens are seen
_ => break,
}
}
}
}
if !is_type_alias {
next = Some(Ok((soft_to_name(tok), *range)));
}
}
}
_ => (), // Not a soft keyword token
} }
} }
@ -111,6 +159,7 @@ fn soft_to_name(tok: &Tok) -> Tok {
let name = match tok { let name = match tok {
Tok::Match => "match", Tok::Match => "match",
Tok::Case => "case", Tok::Case => "case",
Tok::Type => "type",
_ => unreachable!("other tokens never reach here"), _ => unreachable!("other tokens never reach here"),
}; };
Tok::Name { Tok::Name {

2
parser/src/token.rs
View file

@ -188,6 +188,7 @@ pub enum Tok {
Try, Try,
While, While,
Match, Match,
Type,
Case, Case,
With, With,
Yield, Yield,
@ -315,6 +316,7 @@ impl fmt::Display for Tok {
Try => f.write_str("'try'"), Try => f.write_str("'try'"),
While => f.write_str("'while'"), While => f.write_str("'while'"),
Match => f.write_str("'match'"), Match => f.write_str("'match'"),
Type => f.write_str("'type'"),
Case => f.write_str("'case'"), Case => f.write_str("'case'"),
With => f.write_str("'with'"), With => f.write_str("'with'"),
Yield => f.write_str("'yield'"), Yield => f.write_str("'yield'"),