// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar
// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
use crate::{
ast,
error::{LexicalError, LexicalErrorType},
function::{ArgumentList, parse_args, parse_params, validate_arguments},
lexer,
context::set_context,
string::parse_strings,
token::StringKind,
};
use num_bigint::BigInt;
grammar;
// This is a hack to reduce the amount of lalrpop tables generated:
// For each public entry point, a full parse table is generated.
// By having only a single pub function, we reduce this to one.
pub Top: ast::Mod = {
StartModule => ast::Mod::Module { body, type_ignores: vec![] },
StartInteractive => ast::Mod::Interactive { body },
StartExpression ("\n")* => ast::Mod::Expression { body: Box::new(body) },
};
Program: ast::Suite = {
=> {
lines.into_iter().flatten().collect()
},
};
// A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = {
Statement,
"\n" => vec![],
};
Suite: ast::Suite = {
SimpleStatement,
"\n" Indent Dedent => s.into_iter().flatten().collect(),
};
Statement: ast::Suite = {
SimpleStatement,
=> vec![s],
};
SimpleStatement: ast::Suite = {
";"? "\n" => {
let mut statements = vec![s1];
statements.extend(s2.into_iter().map(|e| e.1));
statements
}
};
SmallStatement: ast::Stmt = {
ExpressionStatement,
PassStatement,
DelStatement,
FlowStatement,
ImportStatement,
GlobalStatement,
NonlocalStatement,
AssertStatement,
};
PassStatement: ast::Stmt = {
"pass" => {
ast::Stmt {
location,
end_location: Some(end_location),
custom: (),
node: ast::StmtKind::Pass,
}
},
};
DelStatement: ast::Stmt = {
"del" => {
ast::Stmt {
location,
end_location: Some(end_location),
custom: (),
node: ast::StmtKind::Delete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() },
}
},
};
ExpressionStatement: ast::Stmt = {
=> {
// Just an expression, no assignment:
if suffix.is_empty() {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Expr { value: Box::new(expression) }
}
} else {
let mut targets = vec![set_context(expression, ast::ExprContext::Store)];
let mut values = suffix;
while values.len() > 1 {
targets.push(set_context(values.remove(0), ast::ExprContext::Store));
}
let value = Box::new(values.into_iter().next().unwrap());
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Assign { targets, value, type_comment: None },
}
}
},
=> {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::AugAssign {
target: Box::new(set_context(target, ast::ExprContext::Store)),
op,
value: Box::new(rhs)
},
}
},
> ":" > => {
let simple = matches!(target.node, ast::ExprKind::Name { .. });
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::AnnAssign {
target: Box::new(set_context(target, ast::ExprContext::Store)),
annotation: Box::new(annotation),
value: rhs.map(Box::new),
simple: if simple { 1 } else { 0 },
},
}
},
};
AssignSuffix: ast::Expr = {
"=" => e
};
TestListOrYieldExpr: ast::Expr = {
TestList,
YieldExpr
}
#[inline]
TestOrStarExprList: ast::Expr = {
// as far as I can tell, these were the same
TestList
};
TestOrStarExpr: ast::Expr = {
Test<"all">,
StarExpr,
};
NamedOrStarExpr: ast::Expr = {
NamedExpression,
StarExpr,
};
TestOrStarNamedExpr: ast::Expr = {
NamedExpressionTest,
StarExpr,
};
AugAssign: ast::Operator = {
"+=" => ast::Operator::Add,
"-=" => ast::Operator::Sub,
"*=" => ast::Operator::Mult,
"@=" => ast::Operator::MatMult,
"/=" => ast::Operator::Div,
"%=" => ast::Operator::Mod,
"&=" => ast::Operator::BitAnd,
"|=" => ast::Operator::BitOr,
"^=" => ast::Operator::BitXor,
"<<=" => ast::Operator::LShift,
">>=" => ast::Operator::RShift,
"**=" => ast::Operator::Pow,
"//=" => ast::Operator::FloorDiv,
};
FlowStatement: ast::Stmt = {
"break" => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Break,
}
},
"continue" => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Continue,
}
},
"return" => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Return { value: value.map(Box::new) },
}
},
=> {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Expr { value: Box::new(expression) },
}
},
RaiseStatement,
};
RaiseStatement: ast::Stmt = {
"raise" => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Raise { exc: None, cause: None },
}
},
"raise" > )?> => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
}
},
};
ImportStatement: ast::Stmt = {
"import" >> => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Import { names },
}
},
"from" "import" => {
let (level, module) = source;
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::ImportFrom {
level,
module,
names
},
}
},
};
ImportFromLocation: (Option, Option) = {
=> {
(Some(dots.iter().sum()), Some(name))
},
=> {
(Some(dots.iter().sum()), None)
},
};
ImportDots: usize = {
"..." => 3,
"." => 1,
};
ImportAsNames: Vec = {
>> => i,
"(" >> ","? ")" => i,
"*" => {
// Star import all
vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })]
},
};
#[inline]
ImportAsAlias: ast::Alias = {
=> ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
}
// A name like abc or abc.def.ghi
DottedName: String = {
=> n,
=> {
let mut r = n.to_string();
for x in n2 {
r.push_str(".");
r.push_str(&x.1);
}
r
},
};
GlobalStatement: ast::Stmt = {
"global" > => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Global { names }
}
},
};
NonlocalStatement: ast::Stmt = {
"nonlocal" > => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Nonlocal { names }
}
},
};
AssertStatement: ast::Stmt = {
"assert" > )?> => {
ast::Stmt {
custom: (),
location,
end_location: Some(end_location),
node: ast::StmtKind::Assert {
test: Box::new(test),
msg: msg.map(|e| Box::new(e.1))
}
}
},
};
CompoundStatement: ast::Stmt = {
IfStatement,
WhileStatement,
ForStatement,
TryStatement,
WithStatement,
FuncDef,
ClassDef,
};
IfStatement: ast::Stmt = {
"if" ":" => {
// Determine last else:
let mut last = s3.map(|s| s.2).unwrap_or_default();
let end_location = last
.last()
.or_else(|| s2.last().and_then(|last| last.4.last()))
.or_else(|| body.last())
.unwrap()
.end_location;
// handle elif:
for i in s2.into_iter().rev() {
let x = ast::Stmt {
custom: (),
location: i.0,
end_location,
node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
};
last = vec![x];
}
ast::Stmt {
custom: (),
location,
end_location,
node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
}
},
};
WhileStatement: ast::Stmt = {
"while" ":" => {
let orelse = s2.map(|s| s.2).unwrap_or_default();
let end_location = orelse
.last()
.or_else(|| body.last())
.unwrap()
.end_location;
ast::Stmt {
custom: (),
location,
end_location,
node: ast::StmtKind::While {
test: Box::new(test),
body,
orelse
},
}
},
};
ForStatement: ast::Stmt = {
"for" "in" ":" => {
let orelse = s2.map(|s| s.2).unwrap_or_default();
let end_location = orelse
.last()
.or_else(|| body.last())
.unwrap()
.end_location
.unwrap();
let target = Box::new(set_context(target, ast::ExprContext::Store));
let iter = Box::new(iter);
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment }
} else {
ast::StmtKind::For { target, iter, body, orelse, type_comment }
};
ast::Stmt::new(location, end_location, node)
},
};
TryStatement: ast::Stmt = {
"try" ":" => {
let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default();
let end_location = finalbody
.last()
.map(|last| last.end_location)
.or_else(|| orelse.last().map(|last| last.end_location))
.or_else(|| handlers.last().map(|last| last.end_location))
.unwrap();
ast::Stmt {
custom: (),
location,
end_location,
node: ast::StmtKind::Try {
body,
handlers,
orelse,
finalbody,
},
}
},
"try" ":" => {
let handlers = vec![];
let orelse = vec![];
let finalbody = finally.2;
let end_location = finalbody.last().unwrap().end_location;
ast::Stmt {
custom: (),
location,
end_location,
node: ast::StmtKind::Try {
body,
handlers,
orelse,
finalbody,
},
}
},
};
ExceptClause: ast::Excepthandler = {
"except" ?> ":" => {
let end_location = body.last().unwrap().end_location.unwrap();
ast::Excepthandler::new(
location,
end_location,
ast::ExcepthandlerKind::ExceptHandler {
type_: typ.map(Box::new),
name: None,
body,
},
)
},
"except" "as" Identifier)> ":" => {
let end_location = body.last().unwrap().end_location.unwrap();
ast::Excepthandler::new(
location,
end_location,
ast::ExcepthandlerKind::ExceptHandler {
type_: Some(Box::new(x.0)),
name: Some(x.2),
body,
},
)
},
};
WithStatement: ast::Stmt = {
"with" ":" => {
let end_location = body.last().unwrap().end_location.unwrap();
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncWith { items, body, type_comment }
} else {
ast::StmtKind::With { items, body, type_comment }
};
ast::Stmt::new(location, end_location, node)
},
};
WithItems: Vec = {
"(" ","? ")",
"(" ",")?> > >)*> ","? ")" => {
left.into_iter().flatten().chain([mid]).chain(right).collect()
},
> => vec![<>],
> >)+> => {
[item].into_iter().chain(items).collect()
}
};
#[inline]
WithItemsNoAs: Vec = {
>> => {
<>.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None }).collect()
},
}
WithItem: ast::Withitem = {
> if Goal != "as" => ast::Withitem { context_expr: <>, optional_vars: None },
> "as" > => {
let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
ast::Withitem { context_expr, optional_vars }
},
};
FuncDef: ast::Stmt = {
"def" " Test<"all">)?> ":" => {
let args = Box::new(args);
let returns = r.map(|x| Box::new(x.1));
let end_location = body.last().unwrap().end_location.unwrap();
let type_comment = None;
let node = if is_async.is_some() {
ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment }
} else {
ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment }
};
ast::Stmt::new(location, end_location, node)
},
};
Parameters: ast::Arguments = {
"(" )?> ")" =>? {
let args = validate_arguments(
a.unwrap_or_else(|| ast::Arguments {
posonlyargs: vec![],
args: vec![],
vararg: None,
kwonlyargs: vec![],
kw_defaults: vec![],
kwarg: None,
defaults: vec![]
})
)?;
Ok(args)
}
};
// Note that this is a macro which is used once for function defs, and
// once for lambda defs.
ParameterList: ast::Arguments = {
> )?> ","? =>? {
let (posonlyargs, args, defaults) = parse_params(param1)?;
// Now gather rest of parameters:
let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
Ok(ast::Arguments {
posonlyargs,
args,
kwonlyargs,
vararg,
kwarg,
defaults,
kw_defaults,
})
},
> )> ","? =>? {
let (posonlyargs, args, defaults) = parse_params(param1)?;
// Now gather rest of parameters:
let vararg = None;
let kwonlyargs = vec![];
let kw_defaults = vec![];
let kwarg = kw.1;
Ok(ast::Arguments {
posonlyargs,
args,
kwonlyargs,
vararg,
kwarg,
defaults,
kw_defaults,
})
},
> ","? => {
let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
ast::Arguments {
posonlyargs: vec![],
args: vec![],
kwonlyargs,
vararg,
kwarg,
defaults: vec![],
kw_defaults,
}
},
> ","? => {
ast::Arguments {
posonlyargs: vec![],
args: vec![],
kwonlyargs: vec![],
vararg: None,
kwarg,
defaults: vec![],
kw_defaults: vec![],
}
},
};
// Use inline here to make sure the "," is not creating an ambiguity.
#[inline]
ParameterDefs: (Vec<(ast::Arg, Option)>, Vec<(ast::Arg, Option)>) = {
>> => {
(vec![], args)
},
>> "," "/" )*> => {
(pos_args, args.into_iter().map(|e| e.1).collect())
},
};
ParameterDef: (ast::Arg, Option) = {
=> (i, None),
"=" > => (i, Some(e)),
};
UntypedParameter: ast::Arg = {
=> ast::Arg::new(
location,
end_location,
ast::ArgData { arg, annotation: None, type_comment: None },
),
};
TypedParameter: ast::Arg = {
)?> => {
let annotation = a.map(|x| Box::new(x.1));
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
},
};
// Use inline here to make sure the "," is not creating an ambiguity.
// TODO: figure out another grammar that makes this inline no longer required.
#[inline]
ParameterListStarArgs: (Option>, Vec, Vec, Option>) = {
"*" )*> )?> =>? {
// Extract keyword arguments:
let mut kwonlyargs = Vec::new();
let mut kw_defaults = Vec::new();
let mut kwargs = Vec::new();
for (name, value) in kw.into_iter().map(|x| x.1) {
if let Some(value) = value {
kwonlyargs.push(name);
kw_defaults.push(value);
} else {
kwargs.push(name);
}
}
kwargs.extend(kwonlyargs.into_iter());
if va.is_none() && kwargs.is_empty() && kwarg.is_none() {
Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
location: location,
})?
}
let kwarg = kwarg.map(|n| n.1).flatten();
let va = va.map(Box::new);
Ok((va, kwargs, kw_defaults, kwarg))
}
};
KwargParameter: Option> = {
"**" => {
kwarg.map(Box::new)
}
};
ClassDef: ast::Stmt = {
"class" ":" => {
let (bases, keywords) = match a {
Some((_, arg, _)) => (arg.args, arg.keywords),
None => (vec![], vec![]),
};
let end_location = body.last().unwrap().end_location;
ast::Stmt {
custom: (),
location,
end_location,
node: ast::StmtKind::ClassDef {
name,
bases,
keywords,
body,
decorator_list,
},
}
},
};
// Decorators:
Decorator: ast::Expr = {
"@" "\n" => {
p
},
};
YieldExpr: ast::Expr = {
"yield" => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::Yield { value: value.map(Box::new) }
},
"yield" "from" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::YieldFrom { value: Box::new(e) }
},
};
Test: ast::Expr = {
> "if" > "else" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::IfExp {
test: Box::new(test),
body: Box::new(body),
orelse: Box::new(orelse),
}
},
OrTest,
LambdaDef,
};
NamedExpressionTest: ast::Expr = {
NamedExpression,
Test<"all">,
}
NamedExpression: ast::Expr = {
":=" > => {
ast::Expr {
location,
end_location: value.end_location,
custom: (),
node: ast::ExprKind::NamedExpr {
target: Box::new(ast::Expr::new(
location,
end_location,
ast::ExprKind::Name { id, ctx: ast::ExprContext::Store },
)),
value: Box::new(value),
}
}
},
};
LambdaDef: ast::Expr = {
"lambda" ?> ":" > =>? {
let p = validate_arguments(
p.unwrap_or_else(|| {
ast::Arguments {
posonlyargs: vec![],
args: vec![],
vararg: None,
kwonlyargs: vec![],
kw_defaults: vec![],
kwarg: None,
defaults: vec![]
}
}
))?;
Ok(ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::Lambda {
args: Box::new(p),
body: Box::new(body)
}
})
}
}
OrTest: ast::Expr = {
> )+> => {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
}
},
AndTest,
};
AndTest: ast::Expr = {
> )+> => {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
}
},
NotTest,
};
NotTest: ast::Expr = {
"not" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
},
Comparison,
};
Comparison: ast::Expr = {
> )+> => {
let (ops, comparators) = comparisons.into_iter().unzip();
ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
}
},
Expression,
};
CompOp: ast::Cmpop = {
"==" => ast::Cmpop::Eq,
"!=" => ast::Cmpop::NotEq,
"<" => ast::Cmpop::Lt,
"<=" => ast::Cmpop::LtE,
">" => ast::Cmpop::Gt,
">=" => ast::Cmpop::GtE,
"in" => ast::Cmpop::In,
"not" "in" => ast::Cmpop::NotIn,
"is" => ast::Cmpop::Is,
"is" "not" => ast::Cmpop::IsNot,
};
Expression: ast::Expr = {
> "|" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
},
XorExpression,
};
XorExpression: ast::Expr = {
> "^" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
},
AndExpression,
};
AndExpression: ast::Expr = {
> "&" > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
},
ShiftExpression,
};
ShiftExpression: ast::Expr = {
> > => ast::Expr {
location,
end_location: Some(end_location),
custom: (),
node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
},
ArithmeticExpression,
};
ShiftOp: ast::Operator = {
"<<" => ast::Operator::LShift,
">>" => ast::Operator::RShift,
};
ArithmeticExpression: ast::Expr = {