mirror of
https://github.com/RustPython/Parser.git
synced 2025-07-19 11:05:45 +00:00
Separate byteoffset ast and located ast
This commit is contained in:
Jeong YunWon
parent
f47dfca4e3
commit
a14e43e03a
21 changed files with 893 additions and 562 deletions
|
|
@ -114,8 +114,7 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
|
|||
}
|
||||
|
||||
keywords.push(ast::Keyword::new(
|
||||
start,
|
||||
end,
|
||||
start..end,
|
||||
ast::KeywordData { arg: name, value },
|
||||
));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -211,9 +211,9 @@ pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
|
|||
pub fn lex_located(
|
||||
source: &str,
|
||||
mode: Mode,
|
||||
start_location: TextSize,
|
||||
start_offset: TextSize,
|
||||
) -> impl Iterator<Item = LexResult> + '_ {
|
||||
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_location), mode)
|
||||
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
|
||||
}
|
||||
|
||||
impl<T> Lexer<T>
|
||||
|
|
|
|||
|
|
@ -92,9 +92,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
|
|||
pub fn parse_expression_located(
|
||||
source: &str,
|
||||
path: &str,
|
||||
location: TextSize,
|
||||
offset: TextSize,
|
||||
) -> Result<ast::Expr, ParseError> {
|
||||
parse_located(source, Mode::Expression, path, location).map(|top| match top {
|
||||
parse_located(source, Mode::Expression, path, offset).map(|top| match top {
|
||||
ast::Mod::Expression(ast::ModExpression { body }) => *body,
|
||||
_ => unreachable!(),
|
||||
})
|
||||
|
|
@ -161,9 +161,9 @@ pub fn parse_located(
|
|||
source: &str,
|
||||
mode: Mode,
|
||||
source_path: &str,
|
||||
location: TextSize,
|
||||
offset: TextSize,
|
||||
) -> Result<ast::Mod, ParseError> {
|
||||
let lxr = lexer::lex_located(source, mode, location);
|
||||
let lxr = lexer::lex_located(source, mode, offset);
|
||||
parse_tokens(lxr, mode, source_path)
|
||||
}
|
||||
|
||||
|
|
@ -233,17 +233,17 @@ fn parse_error_from_lalrpop(
|
|||
// TODO: Are there cases where this isn't an EOF?
|
||||
LalrpopError::InvalidToken { location } => ParseError {
|
||||
error: ParseErrorType::Eof,
|
||||
location,
|
||||
offset: location,
|
||||
source_path,
|
||||
},
|
||||
LalrpopError::ExtraToken { token } => ParseError {
|
||||
error: ParseErrorType::ExtraToken(token.1),
|
||||
location: token.0,
|
||||
offset: token.0,
|
||||
source_path,
|
||||
},
|
||||
LalrpopError::User { error } => ParseError {
|
||||
error: ParseErrorType::Lexical(error.error),
|
||||
location: error.location,
|
||||
offset: error.location,
|
||||
source_path,
|
||||
},
|
||||
LalrpopError::UnrecognizedToken { token, expected } => {
|
||||
|
|
@ -252,7 +252,7 @@ fn parse_error_from_lalrpop(
|
|||
let expected = (expected.len() == 1).then(|| expected[0].clone());
|
||||
ParseError {
|
||||
error: ParseErrorType::UnrecognizedToken(token.1, expected),
|
||||
location: token.0,
|
||||
offset: token.0,
|
||||
source_path,
|
||||
}
|
||||
}
|
||||
|
|
@ -262,13 +262,13 @@ fn parse_error_from_lalrpop(
|
|||
if indent_error {
|
||||
ParseError {
|
||||
error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
|
||||
location,
|
||||
offset: location,
|
||||
source_path,
|
||||
}
|
||||
} else {
|
||||
ParseError {
|
||||
error: ParseErrorType::Eof,
|
||||
location,
|
||||
offset: location,
|
||||
source_path,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ SmallStatement: ast::Stmt = {
|
|||
PassStatement: ast::Stmt = {
|
||||
<location:@L> "pass" <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtKind::Pass,
|
||||
)
|
||||
|
|
@ -78,7 +78,7 @@ PassStatement: ast::Stmt = {
|
|||
DelStatement: ast::Stmt = {
|
||||
<location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtDelete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() }.into()
|
||||
)
|
||||
|
|
@ -90,7 +90,7 @@ ExpressionStatement: ast::Stmt = {
|
|||
// Just an expression, no assignment:
|
||||
if suffix.is_empty() {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtExpr { value: Box::new(expression) }.into()
|
||||
)
|
||||
|
|
@ -105,7 +105,7 @@ ExpressionStatement: ast::Stmt = {
|
|||
let value = Box::new(values.into_iter().next().unwrap());
|
||||
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtAssign { targets, value, type_comment: None }.into()
|
||||
)
|
||||
|
|
@ -113,7 +113,7 @@ ExpressionStatement: ast::Stmt = {
|
|||
},
|
||||
<location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtAugAssign {
|
||||
target: Box::new(set_context(target, ast::ExprContext::Store)),
|
||||
|
|
@ -125,7 +125,7 @@ ExpressionStatement: ast::Stmt = {
|
|||
<location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
|
||||
let simple = matches!(target.node, ast::ExprKind::Name { .. });
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtAnnAssign {
|
||||
target: Box::new(set_context(target, ast::ExprContext::Store)),
|
||||
|
|
@ -186,28 +186,28 @@ AugAssign: ast::Operator = {
|
|||
FlowStatement: ast::Stmt = {
|
||||
<location:@L> "break" <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtKind::Break,
|
||||
)
|
||||
},
|
||||
<location:@L> "continue" <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtKind::Continue,
|
||||
)
|
||||
},
|
||||
<location:@L> "return" <value:TestList?> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtReturn { value: value.map(Box::new) }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> <expression:YieldExpr> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtExpr { value: Box::new(expression) }.into()
|
||||
)
|
||||
|
|
@ -218,14 +218,14 @@ FlowStatement: ast::Stmt = {
|
|||
RaiseStatement: ast::Stmt = {
|
||||
<location:@L> "raise" <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtRaise { exc: None, cause: None }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }.into()
|
||||
)
|
||||
|
|
@ -235,7 +235,7 @@ RaiseStatement: ast::Stmt = {
|
|||
ImportStatement: ast::Stmt = {
|
||||
<location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtImport { names }.into()
|
||||
)
|
||||
|
|
@ -243,7 +243,7 @@ ImportStatement: ast::Stmt = {
|
|||
<location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
|
||||
let (level, module) = source;
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtImportFrom {
|
||||
level,
|
||||
|
|
@ -273,14 +273,14 @@ ImportAsNames: Vec<ast::Alias> = {
|
|||
<location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i,
|
||||
<location:@L> "*" <end_location:@R> => {
|
||||
// Star import all
|
||||
vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })]
|
||||
vec![ast::Alias::new(location..end_location, ast::AliasData { name: "*".to_string(), asname: None })]
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
#[inline]
|
||||
ImportAsAlias<I>: ast::Alias = {
|
||||
<location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
|
||||
<location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location..end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
|
||||
}
|
||||
|
||||
// A name like abc or abc.def.ghi
|
||||
|
|
@ -299,7 +299,7 @@ DottedName: String = {
|
|||
GlobalStatement: ast::Stmt = {
|
||||
<location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtGlobal { names }.into()
|
||||
)
|
||||
|
|
@ -309,7 +309,7 @@ GlobalStatement: ast::Stmt = {
|
|||
NonlocalStatement: ast::Stmt = {
|
||||
<location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtNonlocal { names }.into()
|
||||
)
|
||||
|
|
@ -319,7 +319,7 @@ NonlocalStatement: ast::Stmt = {
|
|||
AssertStatement: ast::Stmt = {
|
||||
<location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtAssert {
|
||||
test: Box::new(test),
|
||||
|
|
@ -350,7 +350,7 @@ MatchStatement: ast::Stmt = {
|
|||
.unwrap()
|
||||
.end();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtMatch {
|
||||
subject: Box::new(subject),
|
||||
|
|
@ -367,7 +367,7 @@ MatchStatement: ast::Stmt = {
|
|||
.unwrap()
|
||||
.end();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtMatch {
|
||||
subject: Box::new(subject),
|
||||
|
|
@ -386,11 +386,11 @@ MatchStatement: ast::Stmt = {
|
|||
let mut subjects = subjects;
|
||||
subjects.insert(0, subject);
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtMatch {
|
||||
subject: Box::new(ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple {
|
||||
elts: subjects,
|
||||
|
|
@ -421,7 +421,7 @@ Guard: ast::Expr = {
|
|||
|
||||
Patterns: ast::Pattern = {
|
||||
<location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::PatternMatchSequence {
|
||||
patterns: vec![pattern]
|
||||
|
|
@ -431,7 +431,7 @@ Patterns: ast::Pattern = {
|
|||
let mut patterns = patterns;
|
||||
patterns.insert(0, pattern);
|
||||
ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::PatternMatchSequence {
|
||||
patterns
|
||||
|
|
@ -455,7 +455,7 @@ AsPattern: ast::Pattern = {
|
|||
})?
|
||||
} else {
|
||||
Ok(ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::PatternMatchAs {
|
||||
pattern: Some(Box::new(pattern)),
|
||||
|
|
@ -472,7 +472,7 @@ OrPattern: ast::Pattern = {
|
|||
let mut patterns = patterns;
|
||||
patterns.insert(0, pattern);
|
||||
ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::PatternMatchOr { patterns }.into()
|
||||
)
|
||||
|
|
@ -481,37 +481,37 @@ OrPattern: ast::Pattern = {
|
|||
|
||||
ClosedPattern: ast::Pattern = {
|
||||
<location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
<location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
node,
|
||||
),
|
||||
|
|
@ -543,7 +543,7 @@ StarPattern: ast::PatternKind = {
|
|||
|
||||
ConstantAtom: ast::Expr = {
|
||||
<location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprConstant { value, kind: None }.into()
|
||||
),
|
||||
|
|
@ -552,7 +552,7 @@ ConstantAtom: ast::Expr = {
|
|||
ConstantExpr: ast::Expr = {
|
||||
ConstantAtom,
|
||||
<location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprUnaryOp {
|
||||
op: ast::Unaryop::USub,
|
||||
|
|
@ -563,7 +563,7 @@ ConstantExpr: ast::Expr = {
|
|||
|
||||
AddOpExpr: ast::Expr = {
|
||||
<location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp {
|
||||
left: Box::new(left),
|
||||
|
|
@ -603,7 +603,7 @@ CapturePattern: ast::PatternKind = {
|
|||
|
||||
MatchName: ast::Expr = {
|
||||
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into(),
|
||||
),
|
||||
|
|
@ -611,7 +611,7 @@ MatchName: ast::Expr = {
|
|||
|
||||
MatchNameOrAttr: ast::Expr = {
|
||||
<location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprAttribute {
|
||||
value: Box::new(name),
|
||||
|
|
@ -620,7 +620,7 @@ MatchNameOrAttr: ast::Expr = {
|
|||
}.into(),
|
||||
),
|
||||
<location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprAttribute {
|
||||
value: Box::new(e),
|
||||
|
|
@ -641,7 +641,7 @@ MappingKey: ast::Expr = {
|
|||
AddOpExpr,
|
||||
MatchNameOrAttr,
|
||||
<location:@L> "None" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprConstant {
|
||||
value: ast::Constant::None,
|
||||
|
|
@ -649,7 +649,7 @@ MappingKey: ast::Expr = {
|
|||
}.into(),
|
||||
),
|
||||
<location:@L> "True" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprConstant {
|
||||
value: true.into(),
|
||||
|
|
@ -657,7 +657,7 @@ MappingKey: ast::Expr = {
|
|||
}.into(),
|
||||
),
|
||||
<location:@L> "False" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprConstant {
|
||||
value: false.into(),
|
||||
|
|
@ -804,7 +804,7 @@ IfStatement: ast::Stmt = {
|
|||
// handle elif:
|
||||
for i in s2.into_iter().rev() {
|
||||
let x = ast::Stmt::new(
|
||||
i.0,
|
||||
i.0..
|
||||
end_location,
|
||||
ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last }.into()
|
||||
);
|
||||
|
|
@ -812,7 +812,7 @@ IfStatement: ast::Stmt = {
|
|||
}
|
||||
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtIf { test: Box::new(test), body, orelse: last }.into()
|
||||
)
|
||||
|
|
@ -828,7 +828,7 @@ WhileStatement: ast::Stmt = {
|
|||
.unwrap()
|
||||
.end();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtWhile {
|
||||
test: Box::new(test),
|
||||
|
|
@ -855,7 +855,7 @@ ForStatement: ast::Stmt = {
|
|||
} else {
|
||||
ast::StmtFor { target, iter, body, orelse, type_comment }.into()
|
||||
};
|
||||
ast::Stmt::new(location, end_location, node)
|
||||
ast::Stmt::new(location..end_location, node)
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -870,7 +870,7 @@ TryStatement: ast::Stmt = {
|
|||
.or_else(|| handlers.last().map(|last| last.end()))
|
||||
.unwrap();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtTry {
|
||||
body,
|
||||
|
|
@ -890,7 +890,7 @@ TryStatement: ast::Stmt = {
|
|||
.or_else(|| handlers.last().map(|last| last.end()))
|
||||
.unwrap();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtTryStar {
|
||||
body,
|
||||
|
|
@ -906,7 +906,7 @@ TryStatement: ast::Stmt = {
|
|||
let finalbody = finally.2;
|
||||
let end_location = finalbody.last().unwrap().end();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtTry {
|
||||
body,
|
||||
|
|
@ -922,7 +922,7 @@ ExceptStarClause: ast::Excepthandler = {
|
|||
<location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
|
||||
let end_location = body.last().unwrap().end();
|
||||
ast::Excepthandler::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExcepthandlerExceptHandler {
|
||||
type_: Some(Box::new(typ)),
|
||||
|
|
@ -934,7 +934,7 @@ ExceptStarClause: ast::Excepthandler = {
|
|||
<location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
|
||||
let end_location = body.last().unwrap().end();
|
||||
ast::Excepthandler::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExcepthandlerExceptHandler {
|
||||
type_: Some(Box::new(x.0)),
|
||||
|
|
@ -950,7 +950,7 @@ ExceptClause: ast::Excepthandler = {
|
|||
<location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
|
||||
let end_location = body.last().unwrap().end();
|
||||
ast::Excepthandler::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExcepthandlerExceptHandler {
|
||||
type_: typ.map(Box::new),
|
||||
|
|
@ -962,7 +962,7 @@ ExceptClause: ast::Excepthandler = {
|
|||
<location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
|
||||
let end_location = body.last().unwrap().end();
|
||||
ast::Excepthandler::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExcepthandlerExceptHandler {
|
||||
type_: Some(Box::new(x.0)),
|
||||
|
|
@ -982,7 +982,7 @@ WithStatement: ast::Stmt = {
|
|||
} else {
|
||||
ast::StmtWith { items, body, type_comment }.into()
|
||||
};
|
||||
ast::Stmt::new(location, end_location, node)
|
||||
ast::Stmt::new(location..end_location, node)
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -1023,7 +1023,7 @@ FuncDef: ast::Stmt = {
|
|||
} else {
|
||||
ast::StmtFunctionDef { name, args, body, decorator_list, returns, type_comment }.into()
|
||||
};
|
||||
ast::Stmt::new(location, end_location, node)
|
||||
ast::Stmt::new(location..end_location, node)
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -1126,7 +1126,7 @@ ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
|
|||
|
||||
UntypedParameter: ast::Arg = {
|
||||
<location:@L> <arg:Identifier> <end_location:@R> => ast::Arg::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ArgData { arg, annotation: None, type_comment: None },
|
||||
),
|
||||
|
|
@ -1135,14 +1135,14 @@ UntypedParameter: ast::Arg = {
|
|||
TypedParameter: ast::Arg = {
|
||||
<location:@L> <arg:Identifier> <a:(":" Test<"all">)?> <end_location:@R> => {
|
||||
let annotation = a.map(|x| Box::new(x.1));
|
||||
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
|
||||
ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
|
||||
},
|
||||
};
|
||||
|
||||
StarTypedParameter: ast::Arg = {
|
||||
<location:@L> <arg:Identifier> <a:(":" TestOrStarExpr)?> <end_location:@R> => {
|
||||
let annotation = a.map(|x| Box::new(x.1));
|
||||
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
|
||||
ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -1193,7 +1193,7 @@ ClassDef: ast::Stmt = {
|
|||
};
|
||||
let end_location = body.last().unwrap().end();
|
||||
ast::Stmt::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::StmtClassDef {
|
||||
name,
|
||||
|
|
@ -1215,12 +1215,12 @@ Decorator: ast::Expr = {
|
|||
|
||||
YieldExpr: ast::Expr = {
|
||||
<location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprYield { value: value.map(Box::new) }.into()
|
||||
),
|
||||
<location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprYieldFrom { value: Box::new(e) }.into()
|
||||
),
|
||||
|
|
@ -1228,7 +1228,7 @@ YieldExpr: ast::Expr = {
|
|||
|
||||
Test<Goal>: ast::Expr = {
|
||||
<location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprIfExp {
|
||||
test: Box::new(test),
|
||||
|
|
@ -1248,11 +1248,11 @@ NamedExpressionTest: ast::Expr = {
|
|||
NamedExpression: ast::Expr = {
|
||||
<location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
value.end(),
|
||||
ast::ExprNamedExpr {
|
||||
target: Box::new(ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprName { id, ctx: ast::ExprContext::Store }.into(),
|
||||
)),
|
||||
|
|
@ -1279,7 +1279,7 @@ LambdaDef: ast::Expr = {
|
|||
))?;
|
||||
|
||||
Ok(ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprLambda {
|
||||
args: Box::new(p),
|
||||
|
|
@ -1294,7 +1294,7 @@ OrTest<Goal>: ast::Expr = {
|
|||
let mut values = vec![e1];
|
||||
values.extend(e2.into_iter().map(|e| e.1));
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBoolOp { op: ast::Boolop::Or, values }.into()
|
||||
)
|
||||
|
|
@ -1307,7 +1307,7 @@ AndTest<Goal>: ast::Expr = {
|
|||
let mut values = vec![e1];
|
||||
values.extend(e2.into_iter().map(|e| e.1));
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBoolOp { op: ast::Boolop::And, values }.into()
|
||||
)
|
||||
|
|
@ -1317,7 +1317,7 @@ AndTest<Goal>: ast::Expr = {
|
|||
|
||||
NotTest<Goal>: ast::Expr = {
|
||||
<location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprUnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }.into()
|
||||
),
|
||||
|
|
@ -1328,7 +1328,7 @@ Comparison<Goal>: ast::Expr = {
|
|||
<location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
|
||||
let (ops, comparators) = comparisons.into_iter().unzip();
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprCompare { left: Box::new(left), ops, comparators }.into()
|
||||
)
|
||||
|
|
@ -1351,7 +1351,7 @@ CompOp: ast::Cmpop = {
|
|||
|
||||
Expression<Goal>: ast::Expr = {
|
||||
<location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }.into()
|
||||
),
|
||||
|
|
@ -1360,7 +1360,7 @@ Expression<Goal>: ast::Expr = {
|
|||
|
||||
XorExpression<Goal>: ast::Expr = {
|
||||
<location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }.into()
|
||||
),
|
||||
|
|
@ -1369,7 +1369,7 @@ XorExpression<Goal>: ast::Expr = {
|
|||
|
||||
AndExpression<Goal>: ast::Expr = {
|
||||
<location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }.into()
|
||||
),
|
||||
|
|
@ -1378,7 +1378,7 @@ AndExpression<Goal>: ast::Expr = {
|
|||
|
||||
ShiftExpression<Goal>: ast::Expr = {
|
||||
<location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(e1), op, right: Box::new(e2) }.into()
|
||||
),
|
||||
|
|
@ -1392,7 +1392,7 @@ ShiftOp: ast::Operator = {
|
|||
|
||||
ArithmeticExpression<Goal>: ast::Expr = {
|
||||
<location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
|
||||
),
|
||||
|
|
@ -1406,7 +1406,7 @@ AddOp: ast::Operator = {
|
|||
|
||||
Term<Goal>: ast::Expr = {
|
||||
<location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
|
||||
),
|
||||
|
|
@ -1423,7 +1423,7 @@ MulOp: ast::Operator = {
|
|||
|
||||
Factor<Goal>: ast::Expr = {
|
||||
<location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprUnaryOp { operand: Box::new(e), op }.into()
|
||||
),
|
||||
|
|
@ -1438,7 +1438,7 @@ UnaryOp: ast::Unaryop = {
|
|||
|
||||
Power<Goal>: ast::Expr = {
|
||||
<location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprBinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }.into()
|
||||
),
|
||||
|
|
@ -1448,7 +1448,7 @@ Power<Goal>: ast::Expr = {
|
|||
AtomExpr<Goal>: ast::Expr = {
|
||||
<location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprAwait { value: Box::new(atom) }.into()
|
||||
)
|
||||
|
|
@ -1460,18 +1460,18 @@ AtomExpr2<Goal>: ast::Expr = {
|
|||
Atom<Goal>,
|
||||
<location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprCall { func: Box::new(f), args: a.args, keywords: a.keywords }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprSubscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }.into()
|
||||
),
|
||||
<location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprAttribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }.into()
|
||||
),
|
||||
|
|
@ -1488,7 +1488,7 @@ SubscriptList: ast::Expr = {
|
|||
}
|
||||
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple { elts: dims, ctx: ast::ExprContext::Load }.into(),
|
||||
)
|
||||
|
|
@ -1503,7 +1503,7 @@ Subscript: ast::Expr = {
|
|||
let upper = e2.map(Box::new);
|
||||
let step = e3.flatten().map(Box::new);
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprSlice { lower, upper, step }.into()
|
||||
)
|
||||
|
|
@ -1517,26 +1517,26 @@ SliceOp: Option<ast::Expr> = {
|
|||
Atom<Goal>: ast::Expr = {
|
||||
<location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
|
||||
<location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprConstant { value, kind: None }.into()
|
||||
),
|
||||
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into()
|
||||
),
|
||||
<location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R> => {
|
||||
let elts = e.unwrap_or_default();
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprList { elts, ctx: ast::ExprContext::Load }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprListComp { elt: Box::new(elt), generators }.into()
|
||||
)
|
||||
|
|
@ -1546,7 +1546,7 @@ Atom<Goal>: ast::Expr = {
|
|||
elts.into_iter().next().unwrap()
|
||||
} else {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
|
||||
)
|
||||
|
|
@ -1564,21 +1564,21 @@ Atom<Goal>: ast::Expr = {
|
|||
} else {
|
||||
let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
|
||||
Ok(ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into(),
|
||||
))
|
||||
}
|
||||
},
|
||||
<location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple { elts: Vec::new(), ctx: ast::ExprContext::Load }.into()
|
||||
),
|
||||
"(" <e:YieldExpr> ")" => e,
|
||||
<location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprGeneratorExp { elt: Box::new(elt), generators }.into()
|
||||
)
|
||||
|
|
@ -1596,14 +1596,14 @@ Atom<Goal>: ast::Expr = {
|
|||
.map(|(k, v)| (k.map(|x| *x), v))
|
||||
.unzip();
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprDict { keys, values }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprDictComp {
|
||||
key: Box::new(e1.0),
|
||||
|
|
@ -1613,21 +1613,21 @@ Atom<Goal>: ast::Expr = {
|
|||
)
|
||||
},
|
||||
<location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprSet { elts }.into()
|
||||
),
|
||||
<location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprSetComp { elt: Box::new(elt), generators }.into()
|
||||
)
|
||||
},
|
||||
<location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
|
||||
<location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
|
||||
<location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
|
||||
<location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
|
||||
<location:@L> "True" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
|
||||
<location:@L> "False" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
|
||||
<location:@L> "None" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
|
||||
<location:@L> "..." <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
|
||||
};
|
||||
|
||||
ListLiteralValues: Vec<ast::Expr> = {
|
||||
|
|
@ -1679,7 +1679,7 @@ GenericList<Element>: ast::Expr = {
|
|||
elts.into_iter().next().unwrap()
|
||||
} else {
|
||||
ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
|
||||
)
|
||||
|
|
@ -1690,7 +1690,7 @@ GenericList<Element>: ast::Expr = {
|
|||
// Test
|
||||
StarExpr: ast::Expr = {
|
||||
<location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
|
||||
)
|
||||
|
|
@ -1725,8 +1725,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
|
|||
<location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
|
||||
let expr = match c {
|
||||
Some(c) => ast::Expr::new(
|
||||
location,
|
||||
end_location,
|
||||
location..end_location,
|
||||
ast::ExprGeneratorExp {
|
||||
elt: Box::new(e),
|
||||
generators: c,
|
||||
|
|
@ -1739,7 +1738,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
|
|||
<location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
|
||||
<location:@L> "*" <e:Test<"all">> <end_location:@R> => {
|
||||
let expr = ast::Expr::new(
|
||||
location,
|
||||
location..
|
||||
end_location,
|
||||
ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
|
||||
);
|
||||
|
|
|
|||
279
parser/src/python.rs
generated
279
parser/src/python.rs
generated
File diff suppressed because it is too large
Load diff
|
|
@ -10,9 +10,7 @@ use crate::{
|
|||
token::{StringKind, Tok},
|
||||
};
|
||||
use itertools::Itertools;
|
||||
use rustpython_compiler_core::{
|
||||
text_size::{TextLen, TextSize},
|
||||
};
|
||||
use rustpython_compiler_core::text_size::{TextLen, TextSize};
|
||||
|
||||
// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
|
||||
const MAX_UNICODE_NAME: usize = 88;
|
||||
|
|
@ -67,7 +65,7 @@ impl<'a> StringParser<'a> {
|
|||
|
||||
#[inline]
|
||||
fn expr(&self, node: ExprKind) -> Expr {
|
||||
Expr::new(self.start, self.end, node)
|
||||
Expr::new(self.start..self.end, node)
|
||||
}
|
||||
|
||||
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
|
||||
|
|
@ -624,8 +622,7 @@ pub(crate) fn parse_strings(
|
|||
}
|
||||
}
|
||||
return Ok(Expr::new(
|
||||
initial_start,
|
||||
last_end,
|
||||
initial_start..last_end,
|
||||
ast::ExprConstant {
|
||||
value: Constant::Bytes(content),
|
||||
kind: None,
|
||||
|
|
@ -648,8 +645,7 @@ pub(crate) fn parse_strings(
|
|||
}
|
||||
}
|
||||
return Ok(Expr::new(
|
||||
initial_start,
|
||||
last_end,
|
||||
initial_start..last_end,
|
||||
ast::ExprConstant {
|
||||
value: Constant::Str(content.join("")),
|
||||
kind: initial_kind,
|
||||
|
|
@ -664,8 +660,7 @@ pub(crate) fn parse_strings(
|
|||
|
||||
let take_current = |current: &mut Vec<String>| -> Expr {
|
||||
Expr::new(
|
||||
initial_start,
|
||||
last_end,
|
||||
initial_start..last_end,
|
||||
ast::ExprConstant {
|
||||
value: Constant::Str(current.drain(..).join("")),
|
||||
kind: initial_kind.clone(),
|
||||
|
|
@ -696,8 +691,7 @@ pub(crate) fn parse_strings(
|
|||
}
|
||||
|
||||
Ok(Expr::new(
|
||||
initial_start,
|
||||
last_end,
|
||||
initial_start..last_end,
|
||||
ast::ExprJoinedStr { values: deduped }.into(),
|
||||
))
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue