mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-14 16:45:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Simplify parenthesized context manager parsing with LALRPOP conditions

Browse source 
Signed-off-by: Anders Kaseorg <andersk@mit.edu>
This commit is contained in:
Anders Kaseorg 2022-12-18 17:32:31 -08:00 • committed by Jeong YunWon
parent aeb068d2f1
commit b20571ad0f
2 changed files with 149 additions and 298 deletions
Download patch file Download diff file Expand all files Collapse all files

307
parser/python.lalrpop
View file

@ -11,7 +11,6 @@ use crate::{
context::set_context, context::set_context,
string::parse_strings, string::parse_strings,
token::StringKind, token::StringKind,
with::{ExprOrWithitems, TupleOrWithitems},
}; };
use num_bigint::BigInt; use num_bigint::BigInt;
@ -129,7 +128,7 @@ ExpressionStatement: ast::Stmt = {
}, },
} }
}, },
<location:@L> <target:Test> ":" <annotation:Test> <rhs:AssignSuffix?> <end_location:@R> => { <location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
let simple = matches!(target.node, ast::ExprKind::Name { .. }); let simple = matches!(target.node, ast::ExprKind::Name { .. });
ast::Stmt { ast::Stmt {
custom: (), custom: (),
@ -161,7 +160,7 @@ TestOrStarExprList: ast::Expr = {
}; };
TestOrStarExpr: ast::Expr = { TestOrStarExpr: ast::Expr = {
Test, Test<"all">,
StarExpr, StarExpr,
}; };
@ -170,12 +169,6 @@ TestOrStarNamedExpr: ast::Expr = {
StarExpr, StarExpr,
}; };
TestOrStarNamedExprOrWithitem: (ast::Expr, Option<Box<ast::Expr>>) = {
<e:NamedExpressionTest> => (e, None),
<e:StarExpr> => (e, None),
<e:Test> "as" <v:Expression> => (e, Some(Box::new(v))),
}
AugAssign: ast::Operator = { AugAssign: ast::Operator = {
"+=" => ast::Operator::Add, "+=" => ast::Operator::Add,
"-=" => ast::Operator::Sub, "-=" => ast::Operator::Sub,
@ -237,7 +230,7 @@ RaiseStatement: ast::Stmt = {
node: ast::StmtKind::Raise { exc: None, cause: None }, node: ast::StmtKind::Raise { exc: None, cause: None },
} }
}, },
<location:@L> "raise" <t:Test> <c:("from" Test)?> <end_location:@R> => { <location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
ast::Stmt { ast::Stmt {
custom: (), custom: (),
location, location,
@ -336,7 +329,7 @@ NonlocalStatement: ast::Stmt = {
}; };
AssertStatement: ast::Stmt = { AssertStatement: ast::Stmt = {
<location:@L> "assert" <test:Test> <msg: ("," Test)?> <end_location:@R> => { <location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
ast::Stmt { ast::Stmt {
custom: (), custom: (),
location, location,
@ -473,7 +466,7 @@ TryStatement: ast::Stmt = {
}; };
ExceptClause: ast::Excepthandler = { ExceptClause: ast::Excepthandler = {
<location:@L> "except" <typ:Test?> ":" <body:Suite> => { <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
let end_location = body.last().unwrap().end_location.unwrap(); let end_location = body.last().unwrap().end_location.unwrap();
ast::Excepthandler::new( ast::Excepthandler::new(
location, location,
@ -485,7 +478,7 @@ ExceptClause: ast::Excepthandler = {
}, },
) )
}, },
<location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => { <location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
let end_location = body.last().unwrap().end_location.unwrap(); let end_location = body.last().unwrap().end_location.unwrap();
ast::Excepthandler::new( ast::Excepthandler::new(
location, location,
@ -512,34 +505,34 @@ WithStatement: ast::Stmt = {
}, },
}; };
// These are only used for their types as macro parameters
ExprGoal: ast::Expr = {};
ExprOrWithitemsGoal: ExprOrWithitems = {};
WithItems: Vec<ast::Withitem> = { WithItems: Vec<ast::Withitem> = {
<items:TestAs<ExprOrWithitemsGoal>> =>? items.try_into(), "(" <WithItemsNoAs> ","? ")",
<first:TestAs<ExprOrWithitemsGoal>> "as" <vars:Expression> =>? { "(" <left:(<WithItemsNoAs> ",")?> <mid:WithItem<"as">> <right:("," <WithItem<"all">>)*> ","? ")" => {
let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store))); left.into_iter().flatten().chain([mid]).chain(right).collect()
let context_expr = first.try_into()?;
Ok(vec![ast::Withitem { context_expr, optional_vars }])
}, },
<first:TestAs<ExprOrWithitemsGoal>> <n:("as" Expression)?> "," <mut items:OneOrMore<WithItem>> =>? { <WithItem<"no-withitems">> => vec![<>],
let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store))); <item:WithItem<"all">> <items:("," <WithItem<"all">>)+> => {
let context_expr = first.try_into()?; [item].into_iter().chain(items).collect()
items.insert(0, ast::Withitem { context_expr, optional_vars });
Ok(items)
} }
}; };
WithItem: ast::Withitem = { #[inline]
<context_expr:Test> <n:("as" Expression)?> => { WithItemsNoAs: Vec<ast::Withitem> = {
let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store))); <OneOrMore<NamedExpressionTest>> => {
<>.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None }).collect()
},
}
WithItem<Goal>: ast::Withitem = {
<Test<Goal>> if Goal != "as" => ast::Withitem { context_expr: <>, optional_vars: None },
<context_expr:Test<"all">> "as" <vars:Expression<"all">> => {
let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
ast::Withitem { context_expr, optional_vars } ast::Withitem { context_expr, optional_vars }
}, },
}; };
FuncDef: ast::Stmt = { FuncDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> => { <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test<"all">)?> ":" <body:Suite> => {
let args = Box::new(args); let args = Box::new(args);
let returns = r.map(|x| Box::new(x.1)); let returns = r.map(|x| Box::new(x.1));
let end_location = body.last().unwrap().end_location.unwrap(); let end_location = body.last().unwrap().end_location.unwrap();
@ -643,7 +636,7 @@ ParameterDefs<ArgType>: (Vec<(ast::Arg, Option<ast::Expr>)>, Vec<(ast::Arg, Opti
ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = { ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
<i:ArgType> => (i, None), <i:ArgType> => (i, None),
<i:ArgType> "=" <e:Test> => (i, Some(e)), <i:ArgType> "=" <e:Test<"all">> => (i, Some(e)),
}; };
UntypedParameter: ast::Arg = { UntypedParameter: ast::Arg = {
@ -655,7 +648,7 @@ UntypedParameter: ast::Arg = {
}; };
TypedParameter: ast::Arg = { TypedParameter: ast::Arg = {
<location:@L> <arg:Identifier> <a:(":" Test)?> <end_location:@R> => { <location:@L> <arg:Identifier> <a:(":" Test<"all">)?> <end_location:@R> => {
let annotation = a.map(|x| Box::new(x.1)); let annotation = a.map(|x| Box::new(x.1));
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None }) ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
}, },
@ -729,7 +722,7 @@ YieldExpr: ast::Expr = {
custom: (), custom: (),
node: ast::ExprKind::Yield { value: value.map(Box::new) } node: ast::ExprKind::Yield { value: value.map(Box::new) }
}, },
<location:@L> "yield" "from" <e:Test> <end_location:@R> => ast::Expr { <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
@ -737,9 +730,8 @@ YieldExpr: ast::Expr = {
}, },
}; };
Test = TestAs<ExprGoal>; Test<Goal>: ast::Expr = {
TestAs<Goal>: Goal = { <body:OrTest<"all">> <location:@L> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr {
<body:OrTest> <location:@L> "if" <test:OrTest> "else" <orelse:Test> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
@ -748,13 +740,13 @@ TestAs<Goal>: Goal = {
body: Box::new(body), body: Box::new(body),
orelse: Box::new(orelse), orelse: Box::new(orelse),
} }
}.into(), },
OrTestAs<Goal>, OrTest<Goal>,
<e:LambdaDef> => e.into(), LambdaDef,
}; };
NamedExpressionTest: ast::Expr = { NamedExpressionTest: ast::Expr = {
<location:@L> <left: (Identifier ":=")?> <right:Test> <end_location:@R> => { <location:@L> <left: (Identifier ":=")?> <right:Test<"all">> <end_location:@R> => {
if let Some(l) = left { if let Some(l) = left {
ast::Expr { ast::Expr {
location, location,
@ -776,7 +768,7 @@ NamedExpressionTest: ast::Expr = {
} }
LambdaDef: ast::Expr = { LambdaDef: ast::Expr = {
<location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> <end_location:@R> => { <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> => {
let p = p.unwrap_or_else(|| { let p = p.unwrap_or_else(|| {
ast::Arguments { ast::Arguments {
posonlyargs: vec![], posonlyargs: vec![],
@ -800,9 +792,8 @@ LambdaDef: ast::Expr = {
} }
} }
OrTest = OrTestAs<ExprGoal>; OrTest<Goal>: ast::Expr = {
OrTestAs<Goal>: Goal = { <location:@L> <e1:AndTest<"all">> <e2:("or" AndTest<"all">)+> <end_location:@R> => {
<location:@L> <e1:AndTest> <e2:("or" AndTest)+> <end_location:@R> => {
let mut values = vec![e1]; let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1)); values.extend(e2.into_iter().map(|e| e.1));
ast::Expr { ast::Expr {
@ -810,14 +801,13 @@ OrTestAs<Goal>: Goal = {
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values } node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
}.into() }
}, },
AndTestAs<Goal>, AndTest<Goal>,
}; };
AndTest = AndTestAs<ExprGoal>; AndTest<Goal>: ast::Expr = {
AndTestAs<Goal>: Goal = { <location:@L> <e1:NotTest<"all">> <e2:("and" NotTest<"all">)+> <end_location:@R> => {
<location:@L> <e1:NotTest> <e2:("and" NotTest)+> <end_location:@R> => {
let mut values = vec![e1]; let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1)); values.extend(e2.into_iter().map(|e| e.1));
ast::Expr { ast::Expr {
@ -825,34 +815,32 @@ AndTestAs<Goal>: Goal = {
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values } node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
}.into() }
}, },
NotTestAs<Goal>, NotTest<Goal>,
}; };
NotTest = NotTestAs<ExprGoal>; NotTest<Goal>: ast::Expr = {
NotTestAs<Goal>: Goal = { <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr {
<location:@L> "not" <e:NotTest> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not } node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
}.into(), },
ComparisonAs<Goal>, Comparison<Goal>,
}; };
Comparison = ComparisonAs<ExprGoal>; Comparison<Goal>: ast::Expr = {
ComparisonAs<Goal>: Goal = { <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
<location:@L> <left:Expression> <comparisons:(CompOp Expression)+> <end_location:@R> => {
let (ops, comparators) = comparisons.into_iter().unzip(); let (ops, comparators) = comparisons.into_iter().unzip();
ast::Expr { ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators } node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
}.into() }
}, },
ExpressionAs<Goal>, Expression<Goal>,
}; };
CompOp: ast::Cmpop = { CompOp: ast::Cmpop = {
@ -868,48 +856,44 @@ CompOp: ast::Cmpop = {
"is" "not" => ast::Cmpop::IsNot, "is" "not" => ast::Cmpop::IsNot,
}; };
Expression = ExpressionAs<ExprGoal>; Expression<Goal>: ast::Expr = {
ExpressionAs<Goal>: Goal = { <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr {
<location:@L> <e1:Expression> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) } node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
}.into(), },
XorExpressionAs<Goal>, XorExpression<Goal>,
}; };
XorExpression = XorExpressionAs<ExprGoal>; XorExpression<Goal>: ast::Expr = {
XorExpressionAs<Goal>: Goal = { <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr {
<location:@L> <e1:XorExpression> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) } node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
}.into(), },
AndExpressionAs<Goal>, AndExpression<Goal>,
}; };
AndExpression = AndExpressionAs<ExprGoal>; AndExpression<Goal>: ast::Expr = {
AndExpressionAs<Goal>: Goal = { <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr {
<location:@L> <e1:AndExpression> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) } node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
}.into(), },
ShiftExpressionAs<Goal>, ShiftExpression<Goal>,
}; };
ShiftExpression = ShiftExpressionAs<ExprGoal>; ShiftExpression<Goal>: ast::Expr = {
ShiftExpressionAs<Goal>: Goal = { <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr {
<location:@L> <e1:ShiftExpression> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) } node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
}.into(), },
ArithmeticExpressionAs<Goal>, ArithmeticExpression<Goal>,
}; };
ShiftOp: ast::Operator = { ShiftOp: ast::Operator = {
@ -917,15 +901,14 @@ ShiftOp: ast::Operator = {
">>" => ast::Operator::RShift, ">>" => ast::Operator::RShift,
}; };
ArithmeticExpression = ArithmeticExpressionAs<ExprGoal>; ArithmeticExpression<Goal>: ast::Expr = {
ArithmeticExpressionAs<Goal>: Goal = { <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr {
<location:@L> <a:ArithmeticExpression> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
}.into(), },
TermAs<Goal>, Term<Goal>,
}; };
AddOp: ast::Operator = { AddOp: ast::Operator = {
@ -933,15 +916,14 @@ AddOp: ast::Operator = {
"-" => ast::Operator::Sub, "-" => ast::Operator::Sub,
}; };
Term = TermAs<ExprGoal>; Term<Goal>: ast::Expr = {
TermAs<Goal>: Goal = { <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr {
<location:@L> <a:Term> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
}.into(), },
FactorAs<Goal>, Factor<Goal>,
}; };
MulOp: ast::Operator = { MulOp: ast::Operator = {
@ -952,15 +934,14 @@ MulOp: ast::Operator = {
"@" => ast::Operator::MatMult, "@" => ast::Operator::MatMult,
}; };
Factor = FactorAs<ExprGoal>; Factor<Goal>: ast::Expr = {
FactorAs<Goal>: Goal = { <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R> => ast::Expr {
<location:@L> <op:UnaryOp> <e:Factor> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
}.into(), },
PowerAs<Goal>, Power<Goal>,
}; };
UnaryOp: ast::Unaryop = { UnaryOp: ast::Unaryop = {
@ -969,53 +950,50 @@ UnaryOp: ast::Unaryop = {
"~" => ast::Unaryop::Invert, "~" => ast::Unaryop::Invert,
}; };
Power = PowerAs<ExprGoal>; Power<Goal>: ast::Expr = {
PowerAs<Goal>: Goal = { <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr {
<location:@L> <e:AtomExpr> "**" <b:Factor> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) } node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
}.into(), },
AtomExprAs<Goal>, AtomExpr<Goal>,
}; };
AtomExpr = AtomExprAs<ExprGoal>; AtomExpr<Goal>: ast::Expr = {
AtomExprAs<Goal>: Goal = { <location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
<location:@L> "await" <atom:AtomExpr2> <end_location:@R> => {
ast::Expr { ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Await { value: Box::new(atom) } node: ast::ExprKind::Await { value: Box::new(atom) }
}.into() }
}, },
AtomExpr2As<Goal>, AtomExpr2<Goal>,
} }
AtomExpr2 = AtomExpr2As<ExprGoal>; AtomExpr2<Goal>: ast::Expr = {
AtomExpr2As<Goal>: Goal = { Atom<Goal>,
AtomAs<Goal>, <location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
<location:@L> <f:AtomExpr2> "(" <a:ArgumentList> ")" <end_location:@R> => {
ast::Expr { ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords } node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
}.into() }
}, },
<location:@L> <e:AtomExpr2> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr { <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load } node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
}.into(), },
<location:@L> <e:AtomExpr2> "." <attr:Identifier> <end_location:@R> => ast::Expr { <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load } node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
}.into(), },
}; };
SubscriptList: ast::Expr = { SubscriptList: ast::Expr = {
@ -1039,8 +1017,8 @@ SubscriptList: ast::Expr = {
}; };
Subscript: ast::Expr = { Subscript: ast::Expr = {
Test, Test<"all">,
<e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> <end_location:@R> => { <e1:Test<"all">?> <location:@L> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
let lower = e1.map(Box::new); let lower = e1.map(Box::new);
let upper = e2.map(Box::new); let upper = e2.map(Box::new);
let step = e3.flatten().map(Box::new); let step = e3.flatten().map(Box::new);
@ -1054,24 +1032,23 @@ Subscript: ast::Expr = {
}; };
SliceOp: Option<ast::Expr> = { SliceOp: Option<ast::Expr> = {
<location:@L> ":" <e:Test?> => e, <location:@L> ":" <e:Test<"all">?> => e,
} }
Atom = AtomAs<ExprGoal>; Atom<Goal>: ast::Expr = {
AtomAs<Goal>: Goal = { <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
<location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?.into()),
<location:@L> <value:Constant> <end_location:@R> => ast::Expr { <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Constant { value, kind: None } node: ast::ExprKind::Constant { value, kind: None }
}.into(), },
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr { <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load } node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
}.into(), },
<location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R> => { <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R> => {
let elts = e.unwrap_or_default(); let elts = e.unwrap_or_default();
ast::Expr { ast::Expr {
@ -1079,7 +1056,7 @@ AtomAs<Goal>: Goal = {
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load } node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
}.into() }
}, },
<location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => { <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
ast::Expr { ast::Expr {
@ -1087,40 +1064,48 @@ AtomAs<Goal>: Goal = {
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::ListComp { elt: Box::new(elt), generators } node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
}.into() }
}, },
<location:@L> "(" <items:OneOrMore<TestOrStarNamedExprOrWithitem>> <trailing_comma:","?> ")" <end_location:@R> =>? { <location:@L> "(" <elts:OneOrMore<NamedExpressionTest>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
if items.len() == 1 && items[0].1.is_none() && trailing_comma.is_none() { if elts.len() == 1 && trailing_comma.is_none() {
match items[0].0.node { elts.into_iter().next().unwrap()
ast::ExprKind::Starred { .. } => { } else {
ast::Expr::new(
location,
end_location,
ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
)
}
},
<location:@L> "(" <left:(<OneOrMore<NamedExpressionTest>> ",")?> <mid:StarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
if left.is_none() && right.is_empty() && trailing_comma.is_none() {
Err(LexicalError{ Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: items[0].0.location, location: mid.location,
}.into()) })?
}
_ => {
Ok(items.into_iter().next().unwrap().0.into())
}
}
} else {
TupleOrWithitems { location, end_location, items }.try_into()
} }
let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
Ok(ast::Expr::new(
location,
end_location,
ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
))
}, },
<location:@L> "(" ")" <end_location:@R> => ast::Expr::new( <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
location, location,
end_location, end_location,
ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
).into(), ),
"(" <e:YieldExpr> ")" => e.into(), "(" <e:YieldExpr> ")" => e,
<location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => { <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
ast::Expr { ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators } node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
}.into() }
}, },
"(" <location:@L> "**" <e:Expression> ")" <end_location:@R> =>? { "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
Err(LexicalError{ Err(LexicalError{
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
location: location, location: location,
@ -1170,7 +1155,7 @@ AtomAs<Goal>: Goal = {
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Dict { keys, values } node: ast::ExprKind::Dict { keys, values }
}.into() }
}, },
<location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => { <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
ast::Expr { ast::Expr {
@ -1182,26 +1167,26 @@ AtomAs<Goal>: Goal = {
value: Box::new(e1.1), value: Box::new(e1.1),
generators, generators,
} }
}.into() }
}, },
<location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr { <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::Set { elts } node: ast::ExprKind::Set { elts }
}.into(), },
<location:@L> "{" <elt:Test> <generators:CompFor> "}" <end_location:@R> => { <location:@L> "{" <elt:Test<"all">> <generators:CompFor> "}" <end_location:@R> => {
ast::Expr { ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
node: ast::ExprKind::SetComp { elt: Box::new(elt), generators } node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
}.into() }
}, },
<location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }).into(), <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }),
<location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }).into(), <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }),
<location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }).into(), <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
<location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }).into(), <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
}; };
ListLiteralValues: Vec<ast::Expr> = { ListLiteralValues: Vec<ast::Expr> = {
@ -1213,12 +1198,12 @@ DictLiteralValues: Vec<(Option<Box<ast::Expr>>, ast::Expr)> = {
}; };
DictEntry: (ast::Expr, ast::Expr) = { DictEntry: (ast::Expr, ast::Expr) = {
<e1: Test> ":" <e2: Test> => (e1, e2), <e1: Test<"all">> ":" <e2: Test<"all">> => (e1, e2),
}; };
DictElement: (Option<Box<ast::Expr>>, ast::Expr) = { DictElement: (Option<Box<ast::Expr>>, ast::Expr) = {
<e:DictEntry> => (Some(Box::new(e.0)), e.1), <e:DictEntry> => (Some(Box::new(e.0)), e.1),
"**" <e:Expression> => (None, e), "**" <e:Expression<"all">> => (None, e),
}; };
SetLiteralValues: Vec<ast::Expr> = { SetLiteralValues: Vec<ast::Expr> = {
@ -1226,7 +1211,7 @@ SetLiteralValues: Vec<ast::Expr> = {
}; };
ExpressionOrStarExpression = { ExpressionOrStarExpression = {
Expression, Expression<"all">,
StarExpr StarExpr
}; };
@ -1264,7 +1249,7 @@ GenericList<Element>: ast::Expr = {
// Test // Test
StarExpr: ast::Expr = { StarExpr: ast::Expr = {
<location:@L> "*" <e:Expression> <end_location:@R> => ast::Expr { <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr {
location, location,
end_location: Some(end_location), end_location: Some(end_location),
custom: (), custom: (),
@ -1276,7 +1261,7 @@ StarExpr: ast::Expr = {
CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c; CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
SingleForComprehension: ast::Comprehension = { SingleForComprehension: ast::Comprehension = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> <end_location:@R> => { <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest<"all">> <ifs:ComprehensionIf*> <end_location:@R> => {
let is_async = is_async.is_some(); let is_async = is_async.is_some();
ast::Comprehension { ast::Comprehension {
target: set_context(target, ast::ExprContext::Store), target: set_context(target, ast::ExprContext::Store),
@ -1287,7 +1272,7 @@ SingleForComprehension: ast::Comprehension = {
} }
}; };
ExpressionNoCond: ast::Expr = OrTest; ExpressionNoCond: ast::Expr = OrTest<"all">;
ComprehensionIf: ast::Expr = "if" <c:ExpressionNoCond> => c; ComprehensionIf: ast::Expr = "if" <c:ExpressionNoCond> => c;
ArgumentList: ArgumentList = { ArgumentList: ArgumentList = {
@ -1313,8 +1298,8 @@ FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::
}; };
(None, expr) (None, expr)
}, },
<location:@L> <i:Identifier> "=" <e:Test> <end_location:@R> => (Some((location, end_location, Some(i))), e), <location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
<location:@L> "*" <e:Test> <end_location:@R> => { <location:@L> "*" <e:Test<"all">> <end_location:@R> => {
let expr = ast::Expr::new( let expr = ast::Expr::new(
location, location,
end_location, end_location,
@ -1322,7 +1307,7 @@ FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::
); );
(None, expr) (None, expr)
}, },
<location:@L> "**" <e:Test> <end_location:@R> => (Some((location, end_location, None)), e), <location:@L> "**" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, None)), e),
}; };
#[inline] #[inline]

134
parser/src/with.rs
View file

@ -1,137 +1,3 @@
//! Intermediate types for `with` statement cover grammar.
//!
//! When we start parsing a `with` statement, we don't initially know
//! whether we're looking at a tuple or a Python 3.9+ parenthesized
//! collection of contexts:
//!
//! ```python
//! with (a, b, c) as t: # tuple
//! with (a, b, c): # withitems
//! ```
//!
//! Since LALRPOP requires us to commit to an output type before we
//! have enough information to decide, we build a cover grammar that's
//! convertible either way. This module contains the necessary
//! intermediate data types.
use crate::ast::{self, Location};
use crate::context;
use crate::error::{LexicalError, LexicalErrorType};
use crate::token::Tok;
use lalrpop_util::ParseError as LalrpopError;
/// Represents a parenthesized collection that we might later convert
/// to a tuple or to `with` items.
///
/// It can be converted to either `Expr` or `ExprOrWithitems` with
/// `.try_into()`. The `Expr` conversion will fail if any `as`
/// variables are present. The `ExprOrWithitems` conversion cannot
/// fail (but we need it to have the same interface so we can use
/// LALRPOP macros to declare the cover grammar without much code
/// duplication).
pub struct TupleOrWithitems {
pub location: Location,
pub end_location: Location,
pub items: Vec<(ast::Expr, Option<Box<ast::Expr>>)>,
}
impl TryFrom<TupleOrWithitems> for ast::Expr {
type Error = LalrpopError<Location, Tok, LexicalError>;
fn try_from(tuple_or_withitems: TupleOrWithitems) -> Result<ast::Expr, Self::Error> {
Ok(ast::Expr {
location: tuple_or_withitems.location,
end_location: Some(tuple_or_withitems.end_location),
custom: (),
node: ast::ExprKind::Tuple {
elts: tuple_or_withitems
.items
.into_iter()
.map(|(expr, optional_vars)| {
if let Some(vars) = optional_vars {
Err(LexicalError {
error: LexicalErrorType::OtherError(
"cannot use 'as' here".to_string(),
),
location: vars.location,
})?
}
Ok(expr)
})
.collect::<Result<Vec<ast::Expr>, Self::Error>>()?,
ctx: ast::ExprContext::Load,
},
})
}
}
impl TryFrom<TupleOrWithitems> for ExprOrWithitems {
type Error = LalrpopError<Location, Tok, LexicalError>;
fn try_from(items: TupleOrWithitems) -> Result<ExprOrWithitems, Self::Error> {
Ok(ExprOrWithitems::TupleOrWithitems(items))
}
}
/// Represents either a non-tuple expression, or a parenthesized
/// collection that we might later convert to a tuple or to `with`
/// items.
///
/// It can be constructed from an `Expr` with `.into()`. (The same
/// interface can be used to convert an `Expr` into itself, which is
/// also important for our LALRPOP macro setup.)
///
/// It can be converted to either `Expr` or `Vec<Withitem>` with
/// `.try_into()`. The `Expr` conversion will fail if any `as`
/// clauses are present. The `Vec<Withitem>` conversion will fail if
/// both `as` clauses and starred expressions are present.
pub enum ExprOrWithitems {
Expr(ast::Expr),
TupleOrWithitems(TupleOrWithitems),
}
impl From<ast::Expr> for ExprOrWithitems {
fn from(expr: ast::Expr) -> ExprOrWithitems {
ExprOrWithitems::Expr(expr)
}
}
impl TryFrom<ExprOrWithitems> for ast::Expr {
type Error = LalrpopError<Location, Tok, LexicalError>;
fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<ast::Expr, Self::Error> {
match expr_or_withitems {
ExprOrWithitems::Expr(expr) => Ok(expr),
ExprOrWithitems::TupleOrWithitems(items) => items.try_into(),
}
}
}
impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
type Error = LalrpopError<Location, Tok, LexicalError>;
fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<Vec<ast::Withitem>, Self::Error> {
match expr_or_withitems {
ExprOrWithitems::TupleOrWithitems(tuple_or_withitems)
if !tuple_or_withitems.items.iter().any(|(context_expr, _)| {
matches!(context_expr.node, ast::ExprKind::Starred { .. })
}) =>
{
Ok(tuple_or_withitems
.items
.into_iter()
.map(|(context_expr, optional_vars)| ast::Withitem {
context_expr,
optional_vars: optional_vars.map(|expr| {
Box::new(context::set_context(*expr, ast::ExprContext::Store))
}),
})
.collect())
}
_ => Ok(vec![ast::Withitem {
context_expr: expr_or_withitems.try_into()?,
optional_vars: None,
}]),
}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::parser::parse_program; use crate::parser::parse_program;