RustPython-Parser/parser/src/python.lalrpop

// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar
// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword

use crate::{
    ast::{self as ast, Ranged},
    lexer::{LexicalError, LexicalErrorType},
    function::{ArgumentList, parse_args, parse_params, validate_arguments},
    context::set_context,
    string::parse_strings,
    token::{self, StringKind},
    text_size::TextSize, parser::optional_range
};
use num_bigint::BigInt;

grammar;

// This is a hack to reduce the amount of lalrpop tables generated:
// For each public entry point, a full parse table is generated.
// By having only a single pub function, we reduce this to one.
pub Top: ast::Mod = {
    <start:@L> StartModule <body:Program> <end:@R> => ast::ModModule { body, type_ignores: vec![], range: optional_range(start, end) }.into(),
    <start:@L> StartInteractive <body:Program> <end:@R> => ast::ModInteractive { body, range: optional_range(start, end) }.into(),
    <start:@L> StartExpression <body:TestList> ("\n")* <end:@R> => ast::ModExpression { body: Box::new(body), range: optional_range(start, end) }.into()
};

Program: ast::Suite = {
    <lines:FileLine*> => {
        lines.into_iter().flatten().collect()
    },
};

// A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = {
    Statement,
    "\n" => vec![],
};

Suite: ast::Suite = {
    SimpleStatement,
    "\n" Indent <s:Statement+> Dedent => s.into_iter().flatten().collect(),
};

Statement: ast::Suite = {
    SimpleStatement,
    <s:CompoundStatement> => vec![s],
};

SimpleStatement: ast::Suite = {
    <mut statements:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
        statements.push(last);
        statements
    }
};

SmallStatement: ast::Stmt = {
    ExpressionStatement,
    PassStatement,
    DelStatement,
    FlowStatement,
    ImportStatement,
    GlobalStatement,
    NonlocalStatement,
    AssertStatement,
};

PassStatement: ast::Stmt = {
    <location:@L> "pass" <end_location:@R> => {
        ast::Stmt::Pass(ast::StmtPass { range: (location..end_location).into() })
    },
};

DelStatement: ast::Stmt = {
    <location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
        ast::Stmt::Delete(
            ast::StmtDelete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect(), range: (location..end_location).into()  }
        )
    },
};

ExpressionStatement: ast::Stmt = {
    <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> <end_location:@R> => {
        // Just an expression, no assignment:
        if suffix.is_empty() {
            ast::Stmt::Expr(
                ast::StmtExpr { value: Box::new(expression), range: (location..end_location).into() }
            )
        } else {
            let mut targets = vec![set_context(expression, ast::ExprContext::Store)];
            let mut values = suffix;

            while values.len() > 1 {
                targets.push(set_context(values.remove(0), ast::ExprContext::Store));
            }

            let value = Box::new(values.into_iter().next().unwrap());

            ast::Stmt::Assign(
                ast::StmtAssign { targets, value, type_comment: None, range: (location..end_location).into() }
            )
        }
    },
    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
        ast::Stmt::AugAssign(
            ast::StmtAugAssign {
                target: Box::new(set_context(target, ast::ExprContext::Store)),
                op,
                value: Box::new(rhs),
                range: (location..end_location).into()
            },
        )
    },
    <location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
        let simple = target.is_name_expr();
        ast::Stmt::AnnAssign(
            ast::StmtAnnAssign {
                target: Box::new(set_context(target, ast::ExprContext::Store)),
                annotation: Box::new(annotation),
                value: rhs.map(Box::new),
                simple,
                range: (location..end_location).into()
            },
        )
    },
};

AssignSuffix: ast::Expr = {
    "=" <e:TestListOrYieldExpr> => e
};

TestListOrYieldExpr: ast::Expr = {
    TestList,
    YieldExpr
}

#[inline]
TestOrStarExprList: ast::Expr = {
    // as far as I can tell, these were the same
    TestList
};

TestOrStarExpr: ast::Expr = {
    Test<"all">,
    StarExpr,
};

NamedOrStarExpr: ast::Expr = {
    NamedExpression,
    StarExpr,
};

TestOrStarNamedExpr: ast::Expr = {
    NamedExpressionTest,
    StarExpr,
};

AugAssign: ast::Operator = {
    "+=" => ast::Operator::Add,
    "-=" => ast::Operator::Sub,
    "*=" => ast::Operator::Mult,
    "@=" => ast::Operator::MatMult,
    "/=" => ast::Operator::Div,
    "%=" => ast::Operator::Mod,
    "&=" => ast::Operator::BitAnd,
    "|=" => ast::Operator::BitOr,
    "^=" => ast::Operator::BitXor,
    "<<=" => ast::Operator::LShift,
    ">>=" => ast::Operator::RShift,
    "**=" => ast::Operator::Pow,
    "//=" => ast::Operator::FloorDiv,
};

FlowStatement: ast::Stmt = {
    <location:@L> "break" <end_location:@R> => {

        ast::Stmt::Break(ast::StmtBreak { range: (location..end_location).into() })
    },
    <location:@L> "continue" <end_location:@R> => {
        ast::Stmt::Continue(ast::StmtContinue { range: (location..end_location).into() })
    },
    <location:@L> "return" <value:TestList?> <end_location:@R> => {
        ast::Stmt::Return(
            ast::StmtReturn { value: value.map(Box::new), range: (location..end_location).into() }
        )
    },
    <location:@L> <expression:YieldExpr> <end_location:@R> => {
        ast::Stmt::Expr(
            ast::StmtExpr { value: Box::new(expression), range: (location..end_location).into() }
        )
    },
    RaiseStatement,
};

RaiseStatement: ast::Stmt = {
    <location:@L> "raise" <end_location:@R> => {
        ast::Stmt::Raise(
            ast::StmtRaise { exc: None, cause: None, range: (location..end_location).into() }
        )
    },
    <location:@L> "raise" <t:Test<"all">> <c:("from" <Test<"all">>)?> <end_location:@R> => {
        ast::Stmt::Raise(
            ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x)), range: (location..end_location).into() }
        )
    },
};

ImportStatement: ast::Stmt = {
    <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
        ast::Stmt::Import(
            ast::StmtImport { names, range: (location..end_location).into() }
        )
    },
    <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
        let (level, module) = source;
        ast::Stmt::ImportFrom(
            ast::StmtImportFrom {
                level,
                module,
                names,
                range: (location..end_location).into()
            },
        )
    },
};

ImportFromLocation: (Option<ast::Int>, Option<ast::Identifier>) = {
    <dots: ImportDots*> <name:DottedName> => {
        (Some(ast::Int::new(dots.iter().map(ast::Int::to_u32).sum())), Some(name))
    },
    <dots: ImportDots+> => {
        (Some(ast::Int::new(dots.iter().map(ast::Int::to_u32).sum())), None)
    },
};

ImportDots: ast::Int = {
    "..." => ast::Int::new(3),
    "." => ast::Int::new(1),
};

ImportAsNames: Vec<ast::Alias> = {
    <location:@L> <i:OneOrMore<ImportAsAlias<Identifier>>> <end_location:@R> => i,
    <location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i,
    <location:@L> "*" <end_location:@R> => {
        // Star import all
        vec![ast::Alias { name: ast::Identifier::new("*"), asname: None, range: (location..end_location).into() }]
    },
};


#[inline]
ImportAsAlias<I>: ast::Alias = {
    <location:@L> <name:I> <a: ("as" <Identifier>)?> <end_location:@R> => ast::Alias { name, asname: a, range: (location..end_location).into() },
}

// A name like abc or abc.def.ghi
DottedName: ast::Identifier = {
    <n:name> => ast::Identifier::new(n),
    <n:name> <n2: ("." Identifier)+>  => {
        let mut r = n.to_string();
        for x in n2 {
            r.push('.');
            r.push_str(x.1.as_str());
        }
        ast::Identifier::new(r)
    },
};

GlobalStatement: ast::Stmt = {
    <location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
        ast::Stmt::Global(
            ast::StmtGlobal { names, range: (location..end_location).into() }
        )
    },
};

NonlocalStatement: ast::Stmt = {
    <location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
        ast::Stmt::Nonlocal(
            ast::StmtNonlocal { names, range: (location..end_location).into() }
        )
    },
};

AssertStatement: ast::Stmt = {
    <location:@L> "assert" <test:Test<"all">> <msg: ("," <Test<"all">>)?> <end_location:@R> => {
        ast::Stmt::Assert(
            ast::StmtAssert {
                test: Box::new(test),
                msg: msg.map(|e| Box::new(e)),
                range: (location..end_location).into()
            }
        )
    },
};

CompoundStatement: ast::Stmt = {
    MatchStatement,
    IfStatement,
    WhileStatement,
    ForStatement,
    TryStatement,
    WithStatement,
    FuncDef,
    ClassDef,
};

MatchStatement: ast::Stmt = {
    <location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:MatchCase+> Dedent => {
       let end_location = cases
            .last()
            .unwrap()
            .body
            .last()
            .unwrap()
            .end();
        ast::Stmt::Match(
            ast::StmtMatch {
                subject: Box::new(subject),
                cases,
                range: (location..end_location).into()
            }
        )
    },
    <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent => {
       let end_location = cases
            .last()
            .unwrap()
            .body
            .last()
            .unwrap()
            .end();
        ast::Stmt::Match(
            ast::StmtMatch {
                subject: Box::new(subject),
                cases,
                range: (location..end_location).into()
            }
        )
    },
    <location:@L> "match" <subjects:TwoOrMore<TestOrStarNamedExpr, ",">> ","? ":" "\n" Indent <cases:MatchCase+> Dedent => {
        let end_location = cases
            .last()
            .unwrap()
            .body
            .last()
            .unwrap()
            .end();
        ast::Stmt::Match(
            ast::StmtMatch {
                subject: Box::new(ast::Expr::Tuple(
                    ast::ExprTuple {
                        elts: subjects,
                        ctx: ast::ExprContext::Load,
                        range: (location..end_location).into()
                    },
                )),
                cases,
                range: (location..end_location).into()
            }
        )
    }
}

MatchCase: ast::MatchCase = {
    <start:@L> "case" <pattern:Patterns> <guard:(Guard)?> ":" <body:Suite> <end:@R> => {
        ast::MatchCase {
            pattern,
            guard: guard.map(Box::new),
            body,
            range: optional_range(start, end)
        }
    },
}

Guard: ast::Expr = {
    "if" <guard:NamedExpressionTest> => {
        guard
    }
}

Patterns: ast::Pattern = {
    <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::MatchSequence(
        ast::PatternMatchSequence {
            patterns: vec![pattern],
            range: (location..end_location).into()
        },
    ),
    <location:@L> <patterns:TwoOrMore<Pattern, ",">> ","? <end_location:@R> => {
        ast::Pattern::MatchSequence(
            ast::PatternMatchSequence {
                patterns,
                range: (location..end_location).into()
            },
        )
    },
    <pattern:Pattern> => pattern
}

Pattern: ast::Pattern = {
    <pattern:AsPattern> => pattern,
    <pattern:OrPattern> => pattern,
}

AsPattern: ast::Pattern = {
    <location:@L> <pattern:OrPattern> "as" <name:Identifier> <end_location:@R> =>? {
        if name.as_str() == "_" {
            Err(LexicalError {
                error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()),
                location,
            })?
        } else {
            Ok(ast::Pattern::MatchAs(
                ast::PatternMatchAs {
                   pattern: Some(Box::new(pattern)),
                   name: Some(name),
                   range: (location..end_location).into()
                },
            ))
        }
    },
}

OrPattern: ast::Pattern = {
    <pattern:ClosedPattern> => pattern,
    <location:@L> <patterns:TwoOrMore<ClosedPattern, "|">> <end_location:@R> => {
        ast::Pattern::MatchOr(
            ast::PatternMatchOr { patterns, range: (location..end_location).into() }
        )
    }
}

ClosedPattern: ast::Pattern = {
    <node:LiteralPattern> => node,
    <node:CapturePattern> => node,
    <node:StarPattern> => node,
    <node:ValuePattern> => node,
    <node:SequencePattern> => node,
    <node:MappingPattern> => node,
    <node:ClassPattern> => node,
}

SequencePattern: ast::Pattern = {
    // A single-item tuple is a special case: it's a group pattern, _not_ a sequence pattern.
    <location:@L> "(" <pattern:Pattern> ")" <end_location:@R> => pattern,
    <location:@L> "(" ")" <end_location:@R> => ast::PatternMatchSequence {
        patterns: vec![],
        range: (location..end_location).into()
    }.into(),
    <location:@L> "(" <pattern:Pattern> "," ")" <end_location:@R> => {
            ast::PatternMatchSequence {
                patterns: vec![pattern],
                range: (location..end_location).into()
            }.into()
        },
    <location:@L> "(" <patterns:(<Pattern> ",")+> <last:Pattern> ","? ")" <end_location:@R> => {
        let mut patterns = patterns;
        patterns.push(last);
        ast::PatternMatchSequence {
            patterns,
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> "[" <patterns:Comma<Pattern>> "]" <end_location:@R> => ast::PatternMatchSequence {
        patterns,
        range: (location..end_location).into()
    }.into(),
}

StarPattern: ast::Pattern = {
    <location:@L> "*" <name:Identifier> <end_location:@R> => ast::PatternMatchStar {
        name: if name.as_str() == "_" { None } else { Some(name) },
        range: (location..end_location).into()
    }.into(),
}

ConstantAtom: ast::Expr = {
    <location:@L> <value:Constant> <end_location:@R> => ast::Expr::Constant(
        ast::ExprConstant { value, kind: None, range: (location..end_location).into() }
    ),
}

ConstantExpr: ast::Expr = {
    ConstantAtom,
    <location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr::UnaryOp(
        ast::ExprUnaryOp {
            op: ast::Unaryop::USub,
            operand: Box::new(operand),
            range: (location..end_location).into()
        }
    ),
}

AddOpExpr: ast::Expr = {
    <location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp {
            left: Box::new(left),
            op,
            right: Box::new(right),
            range: (location..end_location).into()
        }
    ),
}

LiteralPattern: ast::Pattern = {
    <location:@L> "None" <end_location:@R> => ast::PatternMatchSingleton {
        value: ast::Constant::None,
        range: (location..end_location).into()
    }.into(),
    <location:@L> "True" <end_location:@R> => ast::PatternMatchSingleton {
        value: true.into(),
        range: (location..end_location).into()
    }.into(),
    <location:@L> "False" <end_location:@R> => ast::PatternMatchSingleton {
        value: false.into(),
        range: (location..end_location).into()
    }.into(),
    <location:@L> <value:ConstantExpr> <end_location:@R> => ast::PatternMatchValue {
        value: Box::new(value),
        range: (location..end_location).into()
    }.into(),
    <location:@L> <value:AddOpExpr> <end_location:@R> => ast::PatternMatchValue {
        value: Box::new(value),
        range: (location..end_location).into()
    }.into(),
    <location:@L> <s:(@L string @R)+> <end_location:@R> =>? Ok(ast::PatternMatchValue {
        value: Box::new(parse_strings(s)?),
        range: (location..end_location).into()
    }.into()),
}

CapturePattern: ast::Pattern = {
    <location:@L> <name:Identifier> <end_location:@R> => ast::PatternMatchAs {
        pattern: None,
        name: if name.as_str() == "_" { None } else { Some(name) },
        range: (location..end_location).into()
    }.into(),
}

MatchName: ast::Expr = {
    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::Name(
        ast::ExprName { id: name, ctx: ast::ExprContext::Load, range: (location..end_location).into() },
    ),
}

MatchNameOrAttr: ast::Expr = {
    <location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr::Attribute(
        ast::ExprAttribute {
            value: Box::new(name),
            attr,
            ctx: ast::ExprContext::Load,
            range: (location..end_location).into()
        },
    ),
    <location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr::Attribute(
        ast::ExprAttribute {
            value: Box::new(e),
            attr,
            ctx: ast::ExprContext::Load,
            range: (location..end_location).into()
        },
    )
}

ValuePattern: ast::Pattern = {
   <location:@L> <e:MatchNameOrAttr> <end_location:@R> => ast::PatternMatchValue {
       value: Box::new(e),
       range: (location..end_location).into()
   }.into(),
}

MappingKey: ast::Expr = {
    ConstantExpr,
    AddOpExpr,
    MatchNameOrAttr,
    <location:@L> "None" <end_location:@R> => ast::Expr::Constant(
        ast::ExprConstant {
            value: ast::Constant::None,
            kind: None,
            range: (location..end_location).into()
        },
    ),
    <location:@L> "True" <end_location:@R> => ast::Expr::Constant(
        ast::ExprConstant {
            value: true.into(),
            kind: None,
            range: (location..end_location).into()
        },
    ),
    <location:@L> "False" <end_location:@R> => ast::Expr::Constant(
        ast::ExprConstant {
            value: false.into(),
            kind: None,
            range: (location..end_location).into()
        },
    ),
    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
}

MatchMappingEntry: (ast::Expr, ast::Pattern) = {
    <k:MappingKey> ":" <v:Pattern> => (k, v),
};

MappingPattern: ast::Pattern = {
    <location:@L> "{" "}" <end_location:@R> => {
        return ast::PatternMatchMapping {
            keys: vec![],
            patterns: vec![],
            rest: None,
            range: (location..end_location).into()
        }.into();
    },
    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","? "}" <end_location:@R> => {
        let (keys, patterns) = e
            .into_iter()
            .unzip();
        return ast::PatternMatchMapping {
            keys,
            patterns,
            rest: None,
            range: (location..end_location).into()
        }.into();
    },
    <location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => {
        return ast::PatternMatchMapping {
            keys: vec![],
            patterns: vec![],
            rest: Some(rest),
            range: (location..end_location).into()
        }.into();
    },
    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","  "**" <rest:Identifier> ","? "}" <end_location:@R> => {
        let (keys, patterns) = e
            .into_iter()
            .unzip();
        return ast::PatternMatchMapping {
            keys,
            patterns,
            rest: Some(rest),
            range: (location..end_location).into()
        }.into();
    },
}

MatchKeywordEntry: (ast::Identifier, ast::Pattern) = {
    <k:Identifier> "=" <v:Pattern> => (k, v),
};

ClassPattern: ast::Pattern = {
    <location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
        let (kwd_attrs, kwd_patterns) = kwds
            .into_iter()
            .unzip();
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns,
            kwd_attrs,
            kwd_patterns,
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => {
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns,
            kwd_attrs: vec![],
            kwd_patterns: vec![],
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchName> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
         let (kwd_attrs, kwd_patterns) = kwds
            .into_iter()
            .unzip();
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns: vec![],
            kwd_attrs,
            kwd_patterns,
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchName> "(" ")" <end_location:@R> => {
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns: vec![],
            kwd_attrs: vec![],
            kwd_patterns: vec![],
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
        let (kwd_attrs, kwd_patterns) = kwds
            .into_iter()
            .unzip();
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns,
            kwd_attrs,
            kwd_patterns,
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => {
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns,
            kwd_attrs: vec![],
            kwd_patterns: vec![],
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchNameOrAttr> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
         let (kwd_attrs, kwd_patterns) = kwds
            .into_iter()
            .unzip();
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns: vec![],
            kwd_attrs,
            kwd_patterns,
            range: (location..end_location).into()
        }.into()
    },
    <location:@L> <e:MatchNameOrAttr> "(" ")" <end_location:@R> => {
        ast::PatternMatchClass {
            cls: Box::new(e),
            patterns: vec![],
            kwd_attrs: vec![],
            kwd_patterns: vec![],
            range: (location..end_location).into()
        }.into()
    },
}

IfStatement: ast::Stmt = {
    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
        // Determine last else:
        let mut last = s3.map(|s| s.2).unwrap_or_default();
        let end_location = last
            .last()
            .or_else(|| s2.last().and_then(|last| last.4.last()))
            .or_else(|| body.last())
            .unwrap()
            .end();
        // handle elif:
        for i in s2.into_iter().rev() {
            let x = ast::Stmt::If(
                ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last, range: (i.0..end_location).into() }
            );
            last = vec![x];
        }

        ast::Stmt::If(
            ast::StmtIf { test: Box::new(test), body, orelse: last, range: (location..end_location).into() }
        )
    },
};

WhileStatement: ast::Stmt = {
    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
        let orelse = s2.map(|s| s.2).unwrap_or_default();
        let end_location = orelse
            .last()
            .or_else(|| body.last())
            .unwrap()
            .end();
        ast::Stmt::While(
            ast::StmtWhile {
                test: Box::new(test),
                body,
                orelse,
                range: (location..end_location).into()
            },
        )
    },
};

ForStatement: ast::Stmt = {
    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
        let orelse = s2.map(|s| s.2).unwrap_or_default();
        let end_location = orelse
            .last()
            .or_else(|| body.last())
            .unwrap()
            .end();
        let target = Box::new(set_context(target, ast::ExprContext::Store));
        let iter = Box::new(iter);
        let type_comment = None;
        if is_async.is_some() {
            ast::Stmt::AsyncFor(ast::StmtAsyncFor { target, iter, body, orelse, type_comment, range: (location..end_location).into() })
        } else {
            ast::Stmt::For(ast::StmtFor { target, iter, body, orelse, type_comment, range: (location..end_location).into() })
        }
    },
};

TryStatement: ast::Stmt = {
    <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
        let orelse = else_suite.map(|s| s.2).unwrap_or_default();
        let finalbody = finally.map(|s| s.2).unwrap_or_default();
        let end_location = finalbody
            .last()
            .map(|last| last.end())
            .or_else(|| orelse.last().map(|last| last.end()))
            .or_else(|| handlers.last().map(|last| last.end()))
            .unwrap();
        ast::Stmt::Try(
            ast::StmtTry {
                body,
                handlers,
                orelse,
                finalbody,
                range: (location..end_location).into()
            },
        )
    },
    <location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
        let orelse = else_suite.map(|s| s.2).unwrap_or_default();
        let finalbody = finally.map(|s| s.2).unwrap_or_default();
        let end_location = finalbody
            .last()
            .or_else(|| orelse.last())
            .map(|last| last.end())
            .or_else(|| handlers.last().map(|last| last.end()))
            .unwrap();
        ast::Stmt::TryStar(
            ast::StmtTryStar {
                body,
                handlers,
                orelse,
                finalbody,
                range: (location..end_location).into()
            },
        )
    },
    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
        let handlers = vec![];
        let orelse = vec![];
        let finalbody = finally.2;
        let end_location = finalbody.last().unwrap().end();
        ast::Stmt::Try(
            ast::StmtTry {
                body,
                handlers,
                orelse,
                finalbody,
                range: (location..end_location).into()
            },
        )
    },
};

ExceptStarClause: ast::Excepthandler = {
    <location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
        let end_location = body.last().unwrap().end();
        ast::Excepthandler::ExceptHandler(
            ast::ExcepthandlerExceptHandler {
                type_: Some(Box::new(typ)),
                name: None,
                body,
                range: (location..end_location).into()
            },
        )
    },
    <location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
        let end_location = body.last().unwrap().end();
        ast::Excepthandler::ExceptHandler(
            ast::ExcepthandlerExceptHandler {
                type_: Some(Box::new(x.0)),
                name: Some(x.2),
                body,
                range: (location..end_location).into()
            },
        )
    },
};


ExceptClause: ast::Excepthandler = {
    <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
        let end_location = body.last().unwrap().end();
        ast::Excepthandler::ExceptHandler(
            ast::ExcepthandlerExceptHandler {
                type_: typ.map(Box::new),
                name: None,
                body,
                range: (location..end_location).into()
            },
        )
    },
    <location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
        let end_location = body.last().unwrap().end();
        ast::Excepthandler::ExceptHandler(
            ast::ExcepthandlerExceptHandler {
                type_: Some(Box::new(x.0)),
                name: Some(x.2),
                body,
                range: (location..end_location).into()
            },
        )
    },
};

WithStatement: ast::Stmt = {
    <location:@L> <is_async:"async"?> "with" <items:WithItems> ":" <body:Suite> => {
        let end_location = body.last().unwrap().end();
        let type_comment = None;
        if is_async.is_some() {
            ast::StmtAsyncWith { items, body, type_comment, range: (location..end_location).into() }.into()
        } else {
            ast::StmtWith { items, body, type_comment, range: (location..end_location).into() }.into()
        }
    },
};

WithItems: Vec<ast::Withitem> = {
    "(" <WithItemsNoAs> ","? ")",
    "(" <left:(<WithItemsNoAs> ",")?> <mid:WithItem<"as">> <right:("," <WithItem<"all">>)*> ","? ")" => {
        left.into_iter().flatten().chain([mid]).chain(right).collect()
    },
    <WithItem<"no-withitems">> => vec![<>],
    <item:WithItem<"all">> <items:("," <WithItem<"all">>)+> => {
        [item].into_iter().chain(items).collect()
    }
};

#[inline]
WithItemsNoAs: Vec<ast::Withitem> = {
    <location:@L> <all:OneOrMore<Test<"all">>> <end_location:@R> => {
        all.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None, range: optional_range(location, end_location) }).collect()
    },
}

WithItem<Goal>: ast::Withitem = {
    <location:@L> <context_expr: Test<Goal>> <end_location:@R> if Goal != "as" => ast::Withitem { context_expr, optional_vars: None, range: optional_range(location, end_location) },
    <location:@L> <context_expr:Test<"all">> "as" <vars:Expression<"all">> <end_location:@R> => {
        let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
        ast::Withitem { context_expr, optional_vars, range: optional_range(location, end_location) }
    },
};

FuncDef: ast::Stmt = {
    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" <Test<"all">>)?> ":" <body:Suite>  => {
        let args = Box::new(args);
        let returns = r.map(|x| Box::new(x));
        let end_location = body.last().unwrap().end();
        let type_comment = None;
        if is_async.is_some() {
            ast::StmtAsyncFunctionDef { name, args, body, decorator_list, returns, type_comment, range: (location..end_location).into() }.into()
        } else {
            ast::StmtFunctionDef { name, args, body, decorator_list, returns, type_comment, range: (location..end_location).into() }.into()
        }
    },
};

Parameters: ast::Arguments = {
    <location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter>)?> ")" <end_location:@R> =>? {
        let args = validate_arguments(
            a.unwrap_or_else(|| ast::Arguments {
                posonlyargs: vec![],
                args: vec![],
                vararg: None,
                kwonlyargs: vec![],
                kw_defaults: vec![],
                kwarg: None,
                defaults: vec![],
                range: optional_range(location, end_location)
            })
        )?;

        Ok(args)
    }
};

// Note that this is a macro which is used once for function defs, and
// once for lambda defs.
ParameterList<ArgType, StarArgType>: ast::Arguments = {
    <location:@L> <param1:ParameterDefs<ArgType>> <args2:("," <ParameterListStarArgs<ArgType, StarArgType>>)?> ","? <end_location:@R> =>? {
        let (posonlyargs, args, defaults) = parse_params(param1)?;

        // Now gather rest of parameters:
        let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.unwrap_or((None, vec![], vec![], None));

        Ok(ast::Arguments {
            posonlyargs,
            args,
            kwonlyargs,
            vararg,
            kwarg,
            defaults,
            kw_defaults,
            range: optional_range(location, end_location)
        })
    },
    <location:@L> <param1:ParameterDefs<ArgType>> <kw:("," <KwargParameter<ArgType>>)> ","? <end_location:@R> =>? {
        let (posonlyargs, args, defaults) = parse_params(param1)?;

        // Now gather rest of parameters:
        let vararg = None;
        let kwonlyargs = vec![];
        let kw_defaults = vec![];
        let kwarg = kw;

        Ok(ast::Arguments {
            posonlyargs,
            args,
            kwonlyargs,
            vararg,
            kwarg,
            defaults,
            kw_defaults,
            range: optional_range(location, end_location)
        })
    },
    <location:@L> <params:ParameterListStarArgs<ArgType, StarArgType>> ","? <end_location:@R> => {
        let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
        ast::Arguments {
            posonlyargs: vec![],
            args: vec![],
            kwonlyargs,
            vararg,
            kwarg,
            defaults: vec![],
            kw_defaults,
            range: optional_range(location, end_location)
        }
    },
    <location:@L> <kwarg:KwargParameter<ArgType>> ","? <end_location:@R> => {
        ast::Arguments {
            posonlyargs: vec![],
            args: vec![],
            kwonlyargs: vec![],
            vararg: None,
            kwarg,
            defaults: vec![],
            kw_defaults: vec![],
            range: optional_range(location, end_location)
        }
    },
};

// Use inline here to make sure the "," is not creating an ambiguity.
#[inline]
ParameterDefs<ArgType>: (Vec<(ast::Arg, Option<ast::Expr>)>, Vec<(ast::Arg, Option<ast::Expr>)>) = {
    <args:OneOrMore<ParameterDef<ArgType>>> => {
        (vec![], args)
    },
    <pos_args:OneOrMore<ParameterDef<ArgType>>> "," "/" <args:("," <ParameterDef<ArgType>>)*>  => {
        (pos_args, args)
    },
};

ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
    <i:ArgType> => (i, None),
    <i:ArgType> "=" <e:Test<"all">> => (i, Some(e)),
};

UntypedParameter: ast::Arg = {
    <location:@L> <arg:Identifier> <end_location:@R> => ast::Arg { arg, annotation: None, type_comment: None, range: (location..end_location).into() },
};

TypedParameter: ast::Arg = {
    <location:@L> <arg:Identifier> <a:(":" <Test<"all">>)?> <end_location:@R> => {
        let annotation = a.map(|x| Box::new(x));
        ast::Arg { arg, annotation, type_comment: None, range: (location..end_location).into() }
    },
};

StarTypedParameter: ast::Arg = {
    <location:@L> <arg:Identifier> <a:(":" <TestOrStarExpr>)?> <end_location:@R> => {
        let annotation = a.map(|x| Box::new(x));
        ast::Arg { arg, annotation, type_comment: None, range: (location..end_location).into() }
    },
};

// Use inline here to make sure the "," is not creating an ambiguity.
// TODO: figure out another grammar that makes this inline no longer required.
#[inline]
ParameterListStarArgs<ArgType, StarArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = {
    <location:@L> "*" <va:StarArgType?> <kw:("," <ParameterDef<ArgType>>)*> <kwarg:("," <KwargParameter<ArgType>>)?> =>? {
        // Extract keyword arguments:
        let mut kwonlyargs = Vec::new();
        let mut kw_defaults = Vec::new();
        let mut kwargs = Vec::with_capacity(kw.len());
        for (name, value) in kw {
            if let Some(value) = value {
                kwonlyargs.push(name);
                kw_defaults.push(value);
            } else {
                kwargs.push(name);
            }
        }
        kwargs.extend(kwonlyargs.into_iter());

        if va.is_none() && kwargs.is_empty() && kwarg.is_none() {
            Err(LexicalError {
                error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
                location,
            })?
        }

        let kwarg = kwarg.flatten();
        let va = va.map(Box::new);

        Ok((va, kwargs, kw_defaults, kwarg))
    }
};

KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
    "**" <kwarg:ArgType?> => {
        kwarg.map(Box::new)
    }
};

ClassDef: ast::Stmt = {
    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
        let (bases, keywords) = match a {
            Some((_, arg, _)) => (arg.args, arg.keywords),
            None => (vec![], vec![]),
        };
        let end_location = body.last().unwrap().end();
        ast::Stmt::ClassDef(
            ast::StmtClassDef {
                name,
                bases,
                keywords,
                body,
                decorator_list,
                range: (location..end_location).into()
            },
        )
    },
};

// Decorators:
Decorator: ast::Expr = {
    <location:@L> "@" <p:NamedExpressionTest> "\n" => {
        p
    },
};

YieldExpr: ast::Expr = {
    <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr::Yield(
        ast::ExprYield { value: value.map(Box::new), range: (location..end_location).into() }
    ),
    <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr::YieldFrom(
        ast::ExprYieldFrom { value: Box::new(e), range: (location..end_location).into() }
    ),
};

Test<Goal>: ast::Expr = {
    <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr::IfExp(
        ast::ExprIfExp {
            test: Box::new(test),
            body: Box::new(body),
            orelse: Box::new(orelse),
            range: (location..end_location).into()
        }
    ),
    OrTest<Goal>,
    LambdaDef,
};

NamedExpressionTest: ast::Expr = {
    NamedExpression,
    Test<"all">,
}

NamedExpression: ast::Expr = {
    <location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
        ast::Expr::NamedExpr(
            ast::ExprNamedExpr {
                target: Box::new(ast::Expr::Name(
                    ast::ExprName { id, ctx: ast::ExprContext::Store, range: (location..end_location).into() },
                )),
                range: (location..value.end()).into(),
                value: Box::new(value),
            }
        )
    },
};

LambdaDef: ast::Expr = {
    <location:@L> "lambda" <p:ParameterList<UntypedParameter, UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> =>? {
        let p = validate_arguments(
            p.unwrap_or_else(|| {
                ast::Arguments {
                    posonlyargs: vec![],
                    args: vec![],
                    vararg: None,
                    kwonlyargs: vec![],
                    kw_defaults: vec![],
                    kwarg: None,
                    defaults: vec![],
                    range: optional_range(location, end_location)
                }
            }
        ))?;

        Ok(ast::Expr::Lambda(
            ast::ExprLambda {
                args: Box::new(p),
                body: Box::new(body),
                range: (location..end_location).into()
            }
        ))
    }
}

OrTest<Goal>: ast::Expr = {
    <location:@L> <mut values:(<AndTest<"all">> "or")+> <last: AndTest<"all">> <end_location:@R> => {
        values.push(last);
        ast::Expr::BoolOp(
            ast::ExprBoolOp { op: ast::Boolop::Or, values, range: (location..end_location).into() }
        )
    },
    AndTest<Goal>,
};

AndTest<Goal>: ast::Expr = {
    <location:@L> <mut values:(<NotTest<"all">> "and")+> <last:NotTest<"all">> <end_location:@R> => {
        values.push(last);
        ast::Expr::BoolOp(
            ast::ExprBoolOp { op: ast::Boolop::And, values, range: (location..end_location).into() }
        )
    },
    NotTest<Goal>,
};

NotTest<Goal>: ast::Expr = {
    <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr::UnaryOp(
        ast::ExprUnaryOp { operand: Box::new(e), op: ast::Unaryop::Not, range: (location..end_location).into() }
    ),
    Comparison<Goal>,
};

Comparison<Goal>: ast::Expr = {
    <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
        let (ops, comparators) = comparisons.into_iter().unzip();
        ast::Expr::Compare(
            ast::ExprCompare { left: Box::new(left), ops, comparators, range: (location..end_location).into() }
        )
    },
    Expression<Goal>,
};

CompOp: ast::Cmpop = {
    "==" => ast::Cmpop::Eq,
    "!=" => ast::Cmpop::NotEq,
    "<" => ast::Cmpop::Lt,
    "<=" => ast::Cmpop::LtE,
    ">" => ast::Cmpop::Gt,
    ">=" => ast::Cmpop::GtE,
    "in" => ast::Cmpop::In,
    "not" "in" => ast::Cmpop::NotIn,
    "is" => ast::Cmpop::Is,
    "is" "not" => ast::Cmpop::IsNot,
};

Expression<Goal>: ast::Expr = {
    <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2), range: (location..end_location).into() }
    ),
    XorExpression<Goal>,
};

XorExpression<Goal>: ast::Expr = {
    <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2), range: (location..end_location).into() }
    ),
    AndExpression<Goal>,
};

AndExpression<Goal>: ast::Expr = {
    <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2), range: (location..end_location).into() }
    ),
    ShiftExpression<Goal>,
};

ShiftExpression<Goal>: ast::Expr = {
    <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(e1), op, right: Box::new(e2), range: (location..end_location).into() }
    ),
    ArithmeticExpression<Goal>,
};

ShiftOp: ast::Operator = {
    "<<" => ast::Operator::LShift,
    ">>" => ast::Operator::RShift,
};

ArithmeticExpression<Goal>: ast::Expr = {
    <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b), range: (location..end_location).into() }
    ),
    Term<Goal>,
};

AddOp: ast::Operator = {
    "+" => ast::Operator::Add,
    "-" => ast::Operator::Sub,
};

Term<Goal>: ast::Expr = {
    <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b), range: (location..end_location).into() }
    ),
    Factor<Goal>,
};

MulOp: ast::Operator = {
    "*" => ast::Operator::Mult,
    "/" => ast::Operator::Div,
    "//" => ast::Operator::FloorDiv,
    "%" => ast::Operator::Mod,
    "@" => ast::Operator::MatMult,
};

Factor<Goal>: ast::Expr = {
    <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R>  => ast::Expr::UnaryOp(
        ast::ExprUnaryOp { operand: Box::new(e), op, range: (location..end_location).into() }
    ),
    Power<Goal>,
};

UnaryOp: ast::Unaryop = {
    "+" => ast::Unaryop::UAdd,
    "-" => ast::Unaryop::USub,
    "~" => ast::Unaryop::Invert,
};

Power<Goal>: ast::Expr = {
    <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr::BinOp(
        ast::ExprBinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b), range: (location..end_location).into() }
    ),
    AtomExpr<Goal>,
};

AtomExpr<Goal>: ast::Expr = {
    <location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
        ast::Expr::Await(
            ast::ExprAwait { value: Box::new(atom), range: (location..end_location).into() }
        )
    },
    AtomExpr2<Goal>,
}

AtomExpr2<Goal>: ast::Expr = {
    Atom<Goal>,
    <location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
        ast::Expr::Call(
            ast::ExprCall { func: Box::new(f), args: a.args, keywords: a.keywords, range: (location..end_location).into() }
        )
    },
    <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr::Subscript(
        ast::ExprSubscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load, range: (location..end_location).into() }
    ),
    <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr::Attribute(
        ast::ExprAttribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load, range: (location..end_location).into() }
    ),
};

SubscriptList: ast::Expr = {
    <location:@L> <s1:Subscript> <end_location:@R> => {
        s1
    },
    <location:@L> <s1:Subscript> "," <end_location:@R> => {
        ast::Expr::Tuple(
            ast::ExprTuple { elts: vec![s1], ctx: ast::ExprContext::Load, range: (location..end_location).into() },
        )
    },
    <location:@L> <elts:TwoOrMore<Subscript, ",">> ","? <end_location:@R> => {
        ast::Expr::Tuple(
            ast::ExprTuple { elts, ctx: ast::ExprContext::Load, range: (location..end_location).into() },
        )
    }
};

Subscript: ast::Expr = {
    TestOrStarNamedExpr,
    <location:@L> <e1:Test<"all">?> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
        let lower = e1.map(Box::new);
        let upper = e2.map(Box::new);
        let step = e3.flatten().map(Box::new);
        ast::Expr::Slice(
            ast::ExprSlice { lower, upper, step, range: (location..end_location).into() }
        )
    }
};

SliceOp: Option<ast::Expr> = {
    <location:@L> ":" <e:Test<"all">?> => e,
}

Atom<Goal>: ast::Expr = {
    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
    <location:@L> <value:Constant> <end_location:@R> => ast::Expr::Constant(
        ast::ExprConstant { value, kind: None, range: (location..end_location).into() }
    ),
    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::Name(
        ast::ExprName { id: name, ctx: ast::ExprContext::Load, range: (location..end_location).into() }
    ),
    <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
        let elts = e.unwrap_or_default();
        ast::Expr::List(
            ast::ExprList { elts, ctx: ast::ExprContext::Load, range: (location..end_location).into() }
        )
    },
    <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
        ast::Expr::ListComp(
            ast::ExprListComp { elt: Box::new(elt), generators, range: (location..end_location).into() }
        )
    },
    <location:@L> "(" <elts:OneOrMore<Test<"all">>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
        if elts.len() == 1 && trailing_comma.is_none() {
            elts.into_iter().next().unwrap()
        } else {
            ast::Expr::Tuple(
                ast::ExprTuple { elts, ctx: ast::ExprContext::Load, range: (location..end_location).into() }
            )
        }
    },
    <location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
        if left.is_none() && right.is_empty() && trailing_comma.is_none() {
            if mid.is_starred_expr() {
                Err(LexicalError{
                    error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
                    location: mid.start(),
                })?
            }
            Ok(mid)
        } else {
            let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
            Ok(ast::Expr::Tuple(
                ast::ExprTuple { elts, ctx: ast::ExprContext::Load, range: (location..end_location).into() },
            ))
        }
    },
    <location:@L> "(" ")" <end_location:@R> => ast::Expr::Tuple(
        ast::ExprTuple { elts: Vec::new(), ctx: ast::ExprContext::Load, range: (location..end_location).into() }
    ),
    "(" <e:YieldExpr> ")" => e,
    <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
        ast::Expr::GeneratorExp(
            ast::ExprGeneratorExp { elt: Box::new(elt), generators, range: (location..end_location).into() }
        )
    },
    "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
        Err(LexicalError{
            error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
            location,
        }.into())
    },
    <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
        let (keys, values) = e
            .unwrap_or_default()
            .into_iter()
            .map(|(k, v)| (k.map(|x| *x), v))
            .unzip();
        ast::Expr::Dict(
            ast::ExprDict { keys, values, range: (location..end_location).into() }
        )
    },
    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
        ast::Expr::DictComp(
            ast::ExprDictComp {
                key: Box::new(e1.0),
                value: Box::new(e1.1),
                generators,
                range: (location..end_location).into()
            }
        )
    },
    <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr::Set(
        ast::ExprSet { elts, range: (location..end_location).into() }
    ),
    <location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
        ast::Expr::SetComp(
            ast::ExprSetComp { elt: Box::new(elt), generators, range: (location..end_location).into() }
        )
    },
    <location:@L> "True" <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: true.into(), kind: None, range: (location..end_location).into() }),
    <location:@L> "False" <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: false.into(), kind: None, range: (location..end_location).into() }),
    <location:@L> "None" <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: ast::Constant::None, kind: None, range: (location..end_location).into() }),
    <location:@L> "..." <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None, range: (location..end_location).into() }),
};

ListLiteralValues: Vec<ast::Expr> = {
    <e:OneOrMore<TestOrStarNamedExpr>> ","? => e,
};

DictLiteralValues: Vec<(Option<Box<ast::Expr>>, ast::Expr)> = {
    <elements:OneOrMore<DictElement>> ","? => elements,
};

DictEntry: (ast::Expr, ast::Expr) = {
    <e1: Test<"all">> ":" <e2: Test<"all">> => (e1, e2),
};

DictElement: (Option<Box<ast::Expr>>, ast::Expr) = {
    <e:DictEntry> => (Some(Box::new(e.0)), e.1),
    "**" <e:Expression<"all">> => (None, e),
};

SetLiteralValues: Vec<ast::Expr> = {
    <e1:OneOrMore<TestOrStarNamedExpr>> ","? => e1
};

ExpressionOrStarExpression = {
    Expression<"all">,
    StarExpr
};

ExpressionList: ast::Expr = {
    GenericList<ExpressionOrStarExpression>
};

ExpressionList2: Vec<ast::Expr> = {
    <elements:OneOrMore<ExpressionOrStarExpression>> ","? => elements,
};

// A test list is one of:
// - a list of expressions
// - a single expression
// - a single expression followed by a trailing comma
#[inline]
TestList: ast::Expr = {
    GenericList<TestOrStarExpr>
};

GenericList<Element>: ast::Expr = {
    <location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> <end_location:@R> => {
        if elts.len() == 1 && trailing_comma.is_none() {
            elts.into_iter().next().unwrap()
        } else {
            ast::Expr::Tuple(
                ast::ExprTuple { elts, ctx: ast::ExprContext::Load, range: (location..end_location).into() }
            )
        }
    }
}

// Test
StarExpr: ast::Expr = {
    <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr::Starred(
        ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load, range: (location..end_location).into() },
    )
};

// Comprehensions:
CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;

SingleForComprehension: ast::Comprehension = {
    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest<"all">> <ifs:ComprehensionIf*> <end_location:@R> => {
        let is_async = is_async.is_some();
        ast::Comprehension {
            target: set_context(target, ast::ExprContext::Store),
            iter,
            ifs,
            is_async,
            range: optional_range(location, end_location)
        }
    }
};

ExpressionNoCond: ast::Expr = OrTest<"all">;
ComprehensionIf: ast::Expr = "if" <c:ExpressionNoCond> => c;

ArgumentList: ArgumentList = {
    <e: Comma<FunctionArgument>> =>? {
        let arg_list = parse_args(e)?;
        Ok(arg_list)
    }
};

FunctionArgument: (Option<(TextSize, TextSize, Option<ast::Identifier>)>, ast::Expr) = {
    <location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
        let expr = match c {
            Some(c) => ast::Expr::GeneratorExp(
                ast::ExprGeneratorExp {
                    elt: Box::new(e),
                    generators: c,
                    range: (location..end_location).into()
                }
            ),
            None => e,
        };
        (None, expr)
    },
    <location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
    <location:@L> "*" <e:Test<"all">> <end_location:@R> => {
        let expr = ast::Expr::Starred(
            ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load, range: (location..end_location).into() },
        );
        (None, expr)
    },
    <location:@L> "**" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, None)), e),
};

/// Comma separated sequence that allows an optional trailing comma.
#[inline]
Comma<T>: Vec<T> = {
    <mut v:(<T> ",")*> <last:T?> => {
        if let Some(element) = last {
            v.push(element);
        }
        v
    }
};

/// One ore more items that are separated by a comma.
OneOrMore<T>: Vec<T> = {
    <e:T> => vec![e],
    <mut v: OneOrMore<T>> "," <e:T> => {
        v.push(e);
        v
    }
};

/// Two or more items that are separted by `Sep`
TwoOrMore<T, Sep>: Vec<T> = {
    <e1:T> Sep <e2:T> => vec![e1, e2],
    <mut v: TwoOrMore<T, Sep>> Sep <e:T> => {
        v.push(e);
        v
    }
};

Constant: ast::Constant = {
    <value:int> => ast::Constant::Int(value),
    <value:float> => ast::Constant::Float(value),
    <s:complex> => ast::Constant::Complex { real: s.0, imag: s.1 },
};

Identifier: ast::Identifier = {
    <s:name> => ast::Identifier::new(s)
};

// Hook external lexer:
extern {
    type Location = TextSize;
    type Error = LexicalError;

    enum token::Tok {
        Indent => token::Tok::Indent,
        Dedent => token::Tok::Dedent,
        StartModule => token::Tok::StartModule,
        StartInteractive => token::Tok::StartInteractive,
        StartExpression => token::Tok::StartExpression,
        "+" => token::Tok::Plus,
        "-" => token::Tok::Minus,
        "~" => token::Tok::Tilde,
        ":" => token::Tok::Colon,
        "." => token::Tok::Dot,
        "..." => token::Tok::Ellipsis,
        "," => token::Tok::Comma,
        "*" => token::Tok::Star,
        "**" => token::Tok::DoubleStar,
        "&" => token::Tok::Amper,
        "@" => token::Tok::At,
        "%" => token::Tok::Percent,
        "//" => token::Tok::DoubleSlash,
        "^" => token::Tok::CircumFlex,
        "|" => token::Tok::Vbar,
        "<<" => token::Tok::LeftShift,
        ">>" => token::Tok::RightShift,
        "/" => token::Tok::Slash,
        "(" => token::Tok::Lpar,
        ")" => token::Tok::Rpar,
        "[" => token::Tok::Lsqb,
        "]" => token::Tok::Rsqb,
        "{" => token::Tok::Lbrace,
        "}" => token::Tok::Rbrace,
        "=" => token::Tok::Equal,
        "+=" => token::Tok::PlusEqual,
        "-=" => token::Tok::MinusEqual,
        "*=" => token::Tok::StarEqual,
        "@=" => token::Tok::AtEqual,
        "/=" => token::Tok::SlashEqual,
        "%=" => token::Tok::PercentEqual,
        "&=" => token::Tok::AmperEqual,
        "|=" => token::Tok::VbarEqual,
        "^=" => token::Tok::CircumflexEqual,
        "<<=" => token::Tok::LeftShiftEqual,
        ">>=" => token::Tok::RightShiftEqual,
        "**=" => token::Tok::DoubleStarEqual,
        "//=" => token::Tok::DoubleSlashEqual,
        ":=" => token::Tok::ColonEqual,
        "==" => token::Tok::EqEqual,
        "!=" => token::Tok::NotEqual,
        "<" => token::Tok::Less,
        "<=" => token::Tok::LessEqual,
        ">" => token::Tok::Greater,
        ">=" => token::Tok::GreaterEqual,
        "->" => token::Tok::Rarrow,
        "and" => token::Tok::And,
        "as" => token::Tok::As,
        "assert" => token::Tok::Assert,
        "async" => token::Tok::Async,
        "await" => token::Tok::Await,
        "break" => token::Tok::Break,
        "class" => token::Tok::Class,
        "continue" => token::Tok::Continue,
        "def" => token::Tok::Def,
        "del" => token::Tok::Del,
        "elif" => token::Tok::Elif,
        "else" => token::Tok::Else,
        "except" => token::Tok::Except,
        "finally" => token::Tok::Finally,
        "for" => token::Tok::For,
        "from" => token::Tok::From,
        "global" => token::Tok::Global,
        "if" => token::Tok::If,
        "import" => token::Tok::Import,
        "in" => token::Tok::In,
        "is" => token::Tok::Is,
        "lambda" => token::Tok::Lambda,
        "nonlocal" => token::Tok::Nonlocal,
        "not" => token::Tok::Not,
        "or" => token::Tok::Or,
        "pass" => token::Tok::Pass,
        "raise" => token::Tok::Raise,
        "return" => token::Tok::Return,
        "try" => token::Tok::Try,
        "while" => token::Tok::While,
        "match" => token::Tok::Match,
        "case" => token::Tok::Case,
        "with" => token::Tok::With,
        "yield" => token::Tok::Yield,
        "True" => token::Tok::True,
        "False" => token::Tok::False,
        "None" => token::Tok::None,
        int => token::Tok::Int { value: <BigInt> },
        float => token::Tok::Float { value: <f64> },
        complex => token::Tok::Complex { real: <f64>, imag: <f64> },
        string => token::Tok::String {
            value: <String>,
            kind: <StringKind>,
            triple_quoted: <bool>
        },
        name => token::Tok::Name { name: <String> },
        "\n" => token::Tok::Newline,
        ";" => token::Tok::Semi,
        // "#" => token::Tok::Comment(_),
    }
}