mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-31 23:57:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add parsing of type alias statements

Browse source
This commit is contained in:
Zanie 2023-07-17 09:12:24 -05:00
parent 6980037ad9
commit f96642aac4
9 changed files with 18961 additions and 18164 deletions
Download patch file Download diff file Expand all files Collapse all files

1
parser/build.rs
View file

@ -154,6 +154,7 @@ fn gen_phf(out_dir: &Path) {
.entry("raise", "Tok::Raise") .entry("raise", "Tok::Raise")
.entry("return", "Tok::Return") .entry("return", "Tok::Return")
.entry("try", "Tok::Try") .entry("try", "Tok::Try")
.entry("type", "Tok::Type")
.entry("while", "Tok::While") .entry("while", "Tok::While")
.entry("with", "Tok::With") .entry("with", "Tok::With")
.entry("yield", "Tok::Yield") .entry("yield", "Tok::Yield")

46
parser/src/parser.rs
View file

@ -636,6 +636,7 @@ class Foo(A, B):
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
} }
#[test] #[test]
#[cfg(feature = "all-nodes-with-ranges")] #[cfg(feature = "all-nodes-with-ranges")]
fn test_parse_class_generic_types() { fn test_parse_class_generic_types() {
@ -894,11 +895,45 @@ except* OSError as e:
assert!(parse(source, Mode::Interactive, "<embedded>").is_ok()); assert!(parse(source, Mode::Interactive, "<embedded>").is_ok());
} }
#[test]
#[cfg(feature = "all-nodes-with-ranges")]
fn test_parse_type_declaration() {
let source = "\
# A non-generic type alias
type IntOrStr = int | str
# A generic type alias
type ListOrSet[T] = list[T] | set[T]
# A type alias that includes a forward reference
type AnimalOrVegetable = Animal | \"Vegetable\"
# A generic self-referential type alias
type RecursiveList[T] = T | list[RecursiveList[T]]
";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
#[cfg(feature = "all-nodes-with-ranges")]
fn test_type_as_identifier() {
let source = r#"\
type = lambda query: query == event
print(type(12))
"#;
use crate::lexer::lex;
let lexer = lex(source, Mode::Module);
println!("tokens {:#?}", lexer.map(|x| x.unwrap().0).collect::<Vec<_>>());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test] #[test]
#[cfg(feature = "all-nodes-with-ranges")] #[cfg(feature = "all-nodes-with-ranges")]
fn test_match_as_identifier() { fn test_match_as_identifier() {
let parse_ast = ast::Suite::parse( let source = r#"\
r#"
match *a + b, c # ((match * a) + b), c match *a + b, c # ((match * a) + b), c
match *(a + b), c # (match * (a + b)), c match *(a + b), c # (match * (a + b)), c
match (*a + b, c) # match ((*(a + b)), c) match (*a + b, c) # match ((*(a + b)), c)
@ -920,11 +955,8 @@ match match:
pass pass
match = lambda query: query == event match = lambda query: query == event
print(match(12)) print(match(12))
"#, "#;
"<test>", insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]

21
parser/src/python.lalrpop
View file

@ -86,6 +86,7 @@ SmallStatement: ast::Stmt = {
GlobalStatement, GlobalStatement,
NonlocalStatement, NonlocalStatement,
AssertStatement, AssertStatement,
TypeAliasStatement,
}; };
PassStatement: ast::Stmt = { PassStatement: ast::Stmt = {
@ -978,6 +979,25 @@ FuncDef: ast::Stmt = {
}, },
}; };
TypeAliasName: ast::Expr = {
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::Name(
ast::ExprName { id: name, ctx: ast::ExprContext::Load, range: (location..end_location).into() },
),
}
TypeAliasStatement: ast::Stmt = {
<location:@L> "type" <name:TypeAliasName> <type_params:TypeParamList?> "=" <value:Test<"all">> <end_location:@R> => {
ast::Stmt::TypeAlias(
ast::StmtTypeAlias {
name: Box::new(name),
value: Box::new(value),
type_params: type_params.unwrap_or_default(),
range: (location..end_location).into()
},
)
},
};
Parameters: ast::Arguments = { Parameters: ast::Arguments = {
<location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter, DoubleStarTypedParameter>)?> ")" <end_location:@R> =>? { <location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter, DoubleStarTypedParameter>)?> ")" <end_location:@R> =>? {
a.as_ref().map(validate_arguments).transpose()?; a.as_ref().map(validate_arguments).transpose()?;
@ -1750,6 +1770,7 @@ extern {
"raise" => token::Tok::Raise, "raise" => token::Tok::Raise,
"return" => token::Tok::Return, "return" => token::Tok::Return,
"try" => token::Tok::Try, "try" => token::Tok::Try,
"type" => token::Tok::Type,
"while" => token::Tok::While, "while" => token::Tok::While,
"match" => token::Tok::Match, "match" => token::Tok::Match,
"case" => token::Tok::Case, "case" => token::Tok::Case,

36405
parser/src/python.rs generated
View file

File diff suppressed because it is too large Load diff

236
parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap generated
View file

@ -1,24 +1,24 @@
--- ---
source: parser/src/parser.rs source: parser/src/parser.rs
expression: parse_ast expression: "ast::Suite::parse(source, \"<test>\").unwrap()"
--- ---
[ [
Expr( Expr(
StmtExpr { StmtExpr {
range: 1..16, range: 2..17,
value: Tuple( value: Tuple(
ExprTuple { ExprTuple {
range: 1..16, range: 2..17,
elts: [ elts: [
BinOp( BinOp(
ExprBinOp { ExprBinOp {
range: 1..13, range: 2..14,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 1..9, range: 2..10,
left: Name( left: Name(
ExprName { ExprName {
range: 1..6, range: 2..7,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -28,7 +28,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 8..9, range: 9..10,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -40,7 +40,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 12..13, range: 13..14,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -51,7 +51,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 15..16, range: 16..17,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -66,17 +66,17 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 42..59, range: 43..60,
value: Tuple( value: Tuple(
ExprTuple { ExprTuple {
range: 42..59, range: 43..60,
elts: [ elts: [
BinOp( BinOp(
ExprBinOp { ExprBinOp {
range: 42..56, range: 43..57,
left: Name( left: Name(
ExprName { ExprName {
range: 42..47, range: 43..48,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -86,10 +86,10 @@ expression: parse_ast
op: Mult, op: Mult,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 50..55, range: 51..56,
left: Name( left: Name(
ExprName { ExprName {
range: 50..51, range: 51..52,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -99,7 +99,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 54..55, range: 55..56,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -112,7 +112,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 58..59, range: 59..60,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -127,13 +127,13 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 85..102, range: 86..103,
value: Call( value: Call(
ExprCall { ExprCall {
range: 85..102, range: 86..103,
func: Name( func: Name(
ExprName { ExprName {
range: 85..90, range: 86..91,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -143,13 +143,13 @@ expression: parse_ast
args: [ args: [
Starred( Starred(
ExprStarred { ExprStarred {
range: 92..98, range: 93..99,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 93..98, range: 94..99,
left: Name( left: Name(
ExprName { ExprName {
range: 93..94, range: 94..95,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -159,7 +159,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 97..98, range: 98..99,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -173,7 +173,7 @@ expression: parse_ast
), ),
Name( Name(
ExprName { ExprName {
range: 100..101, range: 101..102,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -188,16 +188,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 129..145, range: 130..146,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 129..145, range: 130..146,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 129..141, range: 130..142,
left: Name( left: Name(
ExprName { ExprName {
range: 129..134, range: 130..135,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -207,10 +207,10 @@ expression: parse_ast
op: Sub, op: Sub,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 136..141, range: 137..142,
left: Name( left: Name(
ExprName { ExprName {
range: 136..137, range: 137..138,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -220,7 +220,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 140..141, range: 141..142,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -234,7 +234,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 144..145, range: 145..146,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -247,16 +247,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 172..190, range: 173..191,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 172..190, range: 173..191,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 172..186, range: 173..187,
left: Name( left: Name(
ExprName { ExprName {
range: 172..177, range: 173..178,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -266,10 +266,10 @@ expression: parse_ast
op: Sub, op: Sub,
right: BinOp( right: BinOp(
ExprBinOp { ExprBinOp {
range: 180..185, range: 181..186,
left: Name( left: Name(
ExprName { ExprName {
range: 180..181, range: 181..182,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -279,7 +279,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 184..185, range: 185..186,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -293,7 +293,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 189..190, range: 190..191,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -306,19 +306,19 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 217..235, range: 218..236,
value: BinOp( value: BinOp(
ExprBinOp { ExprBinOp {
range: 217..235, range: 218..236,
left: BinOp( left: BinOp(
ExprBinOp { ExprBinOp {
range: 217..231, range: 218..232,
left: Call( left: Call(
ExprCall { ExprCall {
range: 217..227, range: 218..228,
func: Name( func: Name(
ExprName { ExprName {
range: 217..222, range: 218..223,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -328,11 +328,11 @@ expression: parse_ast
args: [ args: [
UnaryOp( UnaryOp(
ExprUnaryOp { ExprUnaryOp {
range: 224..226, range: 225..227,
op: USub, op: USub,
operand: Name( operand: Name(
ExprName { ExprName {
range: 225..226, range: 226..227,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -348,7 +348,7 @@ expression: parse_ast
op: Mult, op: Mult,
right: Name( right: Name(
ExprName { ExprName {
range: 230..231, range: 231..232,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -360,7 +360,7 @@ expression: parse_ast
op: Add, op: Add,
right: Name( right: Name(
ExprName { ExprName {
range: 234..235, range: 235..236,
id: Identifier( id: Identifier(
"c", "c",
), ),
@ -373,16 +373,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 263..273, range: 264..274,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 263..273, range: 264..274,
value: Call( value: Call(
ExprCall { ExprCall {
range: 263..271, range: 264..272,
func: Name( func: Name(
ExprName { ExprName {
range: 263..268, range: 264..269,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -403,16 +403,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 290..302, range: 291..303,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 290..302, range: 291..303,
value: Call( value: Call(
ExprCall { ExprCall {
range: 290..300, range: 291..301,
func: Name( func: Name(
ExprName { ExprName {
range: 290..295, range: 291..296,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -422,7 +422,7 @@ expression: parse_ast
args: [ args: [
Tuple( Tuple(
ExprTuple { ExprTuple {
range: 297..299, range: 298..300,
elts: [], elts: [],
ctx: Load, ctx: Load,
}, },
@ -441,16 +441,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 321..334, range: 322..335,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 321..334, range: 322..335,
value: Call( value: Call(
ExprCall { ExprCall {
range: 321..332, range: 322..333,
func: Name( func: Name(
ExprName { ExprName {
range: 321..326, range: 322..327,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -460,7 +460,7 @@ expression: parse_ast
args: [ args: [
Tuple( Tuple(
ExprTuple { ExprTuple {
range: 328..330, range: 329..331,
elts: [], elts: [],
ctx: Load, ctx: Load,
}, },
@ -479,16 +479,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 353..364, range: 354..365,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 353..364, range: 354..365,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 353..362, range: 354..363,
value: Name( value: Name(
ExprName { ExprName {
range: 353..358, range: 354..359,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -497,7 +497,7 @@ expression: parse_ast
), ),
slice: Name( slice: Name(
ExprName { ExprName {
range: 360..361, range: 361..362,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -517,16 +517,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 382..394, range: 383..395,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 382..394, range: 383..395,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 382..392, range: 383..393,
value: Name( value: Name(
ExprName { ExprName {
range: 382..387, range: 383..388,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -535,11 +535,11 @@ expression: parse_ast
), ),
slice: Tuple( slice: Tuple(
ExprTuple { ExprTuple {
range: 389..391, range: 390..392,
elts: [ elts: [
Name( Name(
ExprName { ExprName {
range: 389..390, range: 390..391,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -563,16 +563,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 435..449, range: 436..450,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 435..449, range: 436..450,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 435..447, range: 436..448,
value: Name( value: Name(
ExprName { ExprName {
range: 435..440, range: 436..441,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -581,11 +581,11 @@ expression: parse_ast
), ),
slice: Tuple( slice: Tuple(
ExprTuple { ExprTuple {
range: 442..446, range: 443..447,
elts: [ elts: [
Name( Name(
ExprName { ExprName {
range: 443..444, range: 444..445,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -609,16 +609,16 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 470..487, range: 471..488,
value: Subscript( value: Subscript(
ExprSubscript { ExprSubscript {
range: 470..487, range: 471..488,
value: Call( value: Call(
ExprCall { ExprCall {
range: 470..477, range: 471..478,
func: Name( func: Name(
ExprName { ExprName {
range: 470..475, range: 471..476,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -631,11 +631,11 @@ expression: parse_ast
), ),
slice: Slice( slice: Slice(
ExprSlice { ExprSlice {
range: 478..486, range: 479..487,
lower: Some( lower: Some(
Name( Name(
ExprName { ExprName {
range: 478..479, range: 479..480,
id: Identifier( id: Identifier(
"a", "a",
), ),
@ -646,7 +646,7 @@ expression: parse_ast
upper: Some( upper: Some(
Name( Name(
ExprName { ExprName {
range: 485..486, range: 486..487,
id: Identifier( id: Identifier(
"b", "b",
), ),
@ -664,13 +664,13 @@ expression: parse_ast
), ),
If( If(
StmtIf { StmtIf {
range: 507..526, range: 508..527,
test: NamedExpr( test: NamedExpr(
ExprNamedExpr { ExprNamedExpr {
range: 510..520, range: 511..521,
target: Name( target: Name(
ExprName { ExprName {
range: 510..515, range: 511..516,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -679,7 +679,7 @@ expression: parse_ast
), ),
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 519..520, range: 520..521,
value: Int( value: Int(
1, 1,
), ),
@ -691,7 +691,7 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 522..526, range: 523..527,
}, },
), ),
], ],
@ -700,10 +700,10 @@ expression: parse_ast
), ),
Match( Match(
StmtMatch { StmtMatch {
range: 527..581, range: 528..582,
subject: Name( subject: Name(
ExprName { ExprName {
range: 533..538, range: 534..539,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -712,13 +712,13 @@ expression: parse_ast
), ),
cases: [ cases: [
MatchCase { MatchCase {
range: 544..556, range: 545..557,
pattern: MatchValue( pattern: MatchValue(
PatternMatchValue { PatternMatchValue {
range: 549..550, range: 550..551,
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 549..550, range: 550..551,
value: Int( value: Int(
1, 1,
), ),
@ -731,19 +731,19 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 552..556, range: 553..557,
}, },
), ),
], ],
}, },
MatchCase { MatchCase {
range: 561..581, range: 562..582,
pattern: MatchValue( pattern: MatchValue(
PatternMatchValue { PatternMatchValue {
range: 566..567, range: 567..568,
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 566..567, range: 567..568,
value: Int( value: Int(
2, 2,
), ),
@ -756,7 +756,7 @@ expression: parse_ast
body: [ body: [
Pass( Pass(
StmtPass { StmtPass {
range: 577..581, range: 578..582,
}, },
), ),
], ],
@ -766,11 +766,11 @@ expression: parse_ast
), ),
Assign( Assign(
StmtAssign { StmtAssign {
range: 582..618, range: 583..619,
targets: [ targets: [
Name( Name(
ExprName { ExprName {
range: 582..587, range: 583..588,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -780,15 +780,15 @@ expression: parse_ast
], ],
value: Lambda( value: Lambda(
ExprLambda { ExprLambda {
range: 590..618, range: 591..619,
args: Arguments { args: Arguments {
range: 597..602, range: 598..603,
posonlyargs: [], posonlyargs: [],
args: [ args: [
ArgWithDefault { ArgWithDefault {
range: 597..602, range: 598..603,
def: Arg { def: Arg {
range: 597..602, range: 598..603,
arg: Identifier( arg: Identifier(
"query", "query",
), ),
@ -804,10 +804,10 @@ expression: parse_ast
}, },
body: Compare( body: Compare(
ExprCompare { ExprCompare {
range: 604..618, range: 605..619,
left: Name( left: Name(
ExprName { ExprName {
range: 604..609, range: 605..610,
id: Identifier( id: Identifier(
"query", "query",
), ),
@ -820,7 +820,7 @@ expression: parse_ast
comparators: [ comparators: [
Name( Name(
ExprName { ExprName {
range: 613..618, range: 614..619,
id: Identifier( id: Identifier(
"event", "event",
), ),
@ -837,13 +837,13 @@ expression: parse_ast
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 619..635, range: 620..636,
value: Call( value: Call(
ExprCall { ExprCall {
range: 619..635, range: 620..636,
func: Name( func: Name(
ExprName { ExprName {
range: 619..624, range: 620..625,
id: Identifier( id: Identifier(
"print", "print",
), ),
@ -853,10 +853,10 @@ expression: parse_ast
args: [ args: [
Call( Call(
ExprCall { ExprCall {
range: 625..634, range: 626..635,
func: Name( func: Name(
ExprName { ExprName {
range: 625..630, range: 626..631,
id: Identifier( id: Identifier(
"match", "match",
), ),
@ -866,7 +866,7 @@ expression: parse_ast
args: [ args: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 631..633, range: 632..634,
value: Int( value: Int(
12, 12,
), ),

242
parser/src/snapshots/rustpython_parser__parser__tests__parse_type_declaration.snap generated Normal file
View file

@ -0,0 +1,242 @@
---
source: parser/src/parser.rs
expression: "ast::Suite::parse(source, \"<test>\").unwrap()"
---
[
TypeAlias(
StmtTypeAlias {
range: 27..52,
name: Name(
ExprName {
range: 32..40,
id: Identifier(
"IntOrStr",
),
ctx: Load,
},
),
type_params: [],
value: BinOp(
ExprBinOp {
range: 43..52,
left: Name(
ExprName {
range: 43..46,
id: Identifier(
"int",
),
ctx: Load,
},
),
op: BitOr,
right: Name(
ExprName {
range: 49..52,
id: Identifier(
"str",
),
ctx: Load,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 77..113,
name: Name(
ExprName {
range: 82..91,
id: Identifier(
"ListOrSet",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 92..93,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: BinOp(
ExprBinOp {
range: 97..113,
left: Subscript(
ExprSubscript {
range: 97..104,
value: Name(
ExprName {
range: 97..101,
id: Identifier(
"list",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 102..103,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
op: BitOr,
right: Subscript(
ExprSubscript {
range: 107..113,
value: Name(
ExprName {
range: 107..110,
id: Identifier(
"set",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 111..112,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 164..209,
name: Name(
ExprName {
range: 169..186,
id: Identifier(
"AnimalOrVegetable",
),
ctx: Load,
},
),
type_params: [],
value: BinOp(
ExprBinOp {
range: 189..209,
left: Name(
ExprName {
range: 189..195,
id: Identifier(
"Animal",
),
ctx: Load,
},
),
op: BitOr,
right: Constant(
ExprConstant {
range: 198..209,
value: Str(
"Vegetable",
),
kind: None,
},
),
},
),
},
),
TypeAlias(
StmtTypeAlias {
range: 251..301,
name: Name(
ExprName {
range: 256..269,
id: Identifier(
"RecursiveList",
),
ctx: Load,
},
),
type_params: [
TypeVar(
TypeParamTypeVar {
range: 270..271,
name: Identifier(
"T",
),
bound: None,
},
),
],
value: BinOp(
ExprBinOp {
range: 275..301,
left: Name(
ExprName {
range: 275..276,
id: Identifier(
"T",
),
ctx: Load,
},
),
op: BitOr,
right: Subscript(
ExprSubscript {
range: 279..301,
value: Name(
ExprName {
range: 279..283,
id: Identifier(
"list",
),
ctx: Load,
},
),
slice: Subscript(
ExprSubscript {
range: 284..300,
value: Name(
ExprName {
range: 284..297,
id: Identifier(
"RecursiveList",
),
ctx: Load,
},
),
slice: Name(
ExprName {
range: 298..299,
id: Identifier(
"T",
),
ctx: Load,
},
),
ctx: Load,
},
),
ctx: Load,
},
),
},
),
},
),
]

125
parser/src/snapshots/rustpython_parser__parser__tests__type_as_identifier.snap generated Normal file
View file

@ -0,0 +1,125 @@
---
source: parser/src/parser.rs
expression: "ast::Suite::parse(source, \"<test>\").unwrap()"
---
[
Assign(
StmtAssign {
range: 2..37,
targets: [
Name(
ExprName {
range: 2..6,
id: Identifier(
"type",
),
ctx: Store,
},
),
],
value: Lambda(
ExprLambda {
range: 9..37,
args: Arguments {
range: 16..21,
posonlyargs: [],
args: [
ArgWithDefault {
range: 16..21,
def: Arg {
range: 16..21,
arg: Identifier(
"query",
),
annotation: None,
type_comment: None,
},
default: None,
},
],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
body: Compare(
ExprCompare {
range: 23..37,
left: Name(
ExprName {
range: 23..28,
id: Identifier(
"query",
),
ctx: Load,
},
),
ops: [
Eq,
],
comparators: [
Name(
ExprName {
range: 32..37,
id: Identifier(
"event",
),
ctx: Load,
},
),
],
},
),
},
),
type_comment: None,
},
),
Expr(
StmtExpr {
range: 38..53,
value: Call(
ExprCall {
range: 38..53,
func: Name(
ExprName {
range: 38..43,
id: Identifier(
"print",
),
ctx: Load,
},
),
args: [
Call(
ExprCall {
range: 44..52,
func: Name(
ExprName {
range: 44..48,
id: Identifier(
"type",
),
ctx: Load,
},
),
args: [
Constant(
ExprConstant {
range: 49..51,
value: Int(
12,
),
kind: None,
},
),
],
keywords: [],
},
),
],
keywords: [],
},
),
},
),
]

47
parser/src/soft_keywords.rs
View file

@ -2,14 +2,17 @@ use crate::{lexer::LexResult, token::Tok, Mode};
use itertools::{Itertools, MultiPeek}; use itertools::{Itertools, MultiPeek};
/// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match` /// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match`
/// and `case`). /// `case`, and `type`).
/// ///
/// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords /// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords
/// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain /// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain
/// contexts. /// contexts.
/// ///
/// Later, [PEP 695](https://peps.python.org/pep-0695/#generic-type-alias) introduced the `type`
/// soft keyword.
///
/// This function modifies a token stream to accommodate this change. In particular, it replaces /// This function modifies a token stream to accommodate this change. In particular, it replaces
/// `match` and `case` tokens with `identifier` tokens if they are used as identifiers. /// soft keyword tokens with `identifier` tokens if they are used as identifiers.
/// ///
/// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and /// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
/// parser, as neither of them need to be aware of soft keywords. /// parser, as neither of them need to be aware of soft keywords.
@ -43,14 +46,16 @@ where
fn next(&mut self) -> Option<LexResult> { fn next(&mut self) -> Option<LexResult> {
let mut next = self.underlying.next(); let mut next = self.underlying.next();
if let Some(Ok((tok, range))) = next.as_ref() { if let Some(Ok((tok, range))) = next.as_ref() {
// If the token is a `match` or `case` token, check if it's used as an identifier. // If the token is a soft keyword e.g. `type`, `match`, or `case`, check if it's
// We assume every `match` or `case` is an identifier unless both of the following // used as an identifier. We assume every soft keyword use is an identifier unless
// conditions are met: // a heuristic is met.
// For `match` and `case`, all of the following conditions must be met:
// 1. The token is at the start of a logical line. // 1. The token is at the start of a logical line.
// 2. The logical line contains a top-level colon (that is, a colon that is not nested // 2. The logical line contains a top-level colon (that is, a colon that is not nested
// inside a parenthesized expression, list, or dictionary). // inside a parenthesized expression, list, or dictionary).
// 3. The top-level colon is not the immediate sibling of a `match` or `case` token. // 3. The top-level colon is not the immediate sibling of a soft keyword token.
// (This is to avoid treating `match` and `case` as identifiers when annotated with // (This is to avoid treating soft keywords as identifiers when annotated with
// type hints.) // type hints.)
if matches!(tok, Tok::Match | Tok::Case) { if matches!(tok, Tok::Match | Tok::Case) {
if !self.start_of_line { if !self.start_of_line {
@ -82,6 +87,33 @@ where
} }
} }
} }
// For `type` all of the following conditions must be met:
// 1. The token is at the start of a logical line.
// 2. The type token is followed by a name token.
// 3. The name token is followed by an equality token.
else if matches!(tok, Tok::Type) {
if !self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0;
let mut seen_name = false;
let mut seen_equal = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Name { .. } if nesting == 0 => seen_name = true,
Tok::Equal if nesting == 0 && seen_name => seen_equal = true,
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
}
}
if !(seen_name && seen_equal) {
next = Some(Ok((soft_to_name(tok), *range)));
}
}
}
} }
self.start_of_line = next.as_ref().map_or(false, |lex_result| { self.start_of_line = next.as_ref().map_or(false, |lex_result| {
@ -111,6 +143,7 @@ fn soft_to_name(tok: &Tok) -> Tok {
let name = match tok { let name = match tok {
Tok::Match => "match", Tok::Match => "match",
Tok::Case => "case", Tok::Case => "case",
Tok::Type => "type",
_ => unreachable!("other tokens never reach here"), _ => unreachable!("other tokens never reach here"),
}; };
Tok::Name { Tok::Name {

2
parser/src/token.rs
View file

@ -188,6 +188,7 @@ pub enum Tok {
Try, Try,
While, While,
Match, Match,
Type,
Case, Case,
With, With,
Yield, Yield,
@ -315,6 +316,7 @@ impl fmt::Display for Tok {
Try => f.write_str("'try'"), Try => f.write_str("'try'"),
While => f.write_str("'while'"), While => f.write_str("'while'"),
Match => f.write_str("'match'"), Match => f.write_str("'match'"),
Type => f.write_str("'type'"),
Case => f.write_str("'case'"), Case => f.write_str("'case'"),
With => f.write_str("'with'"), With => f.write_str("'with'"),
Yield => f.write_str("'yield'"), Yield => f.write_str("'yield'"),