diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py index 71c9032..ea035ed 100755 --- a/ast/asdl_rs.py +++ b/ast/asdl_rs.py @@ -304,7 +304,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor): depth, ) self.emit( - "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? })", + "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, end_location: node.end_location, node: f(folder, node.node)? })", depth + 1, ) self.emit("}", depth) @@ -617,9 +617,6 @@ class TraitImplVisitor(EmitVisitor): column = self.decode_field(asdl.Field("int", "col_offset"), typename) self.emit(f"let _location = ast::Location::new({row}, {column});", depth) - def wrap_located_node(self, depth): - self.emit(f"let node = ast::Located::new(_location, node);", depth) - def decode_field(self, field, typename): name = json.dumps(field.name) if field.opt and not field.seq: @@ -658,13 +655,14 @@ def write_ast_def(mod, typeinfo, f): """ pub struct Located { pub location: Location, + pub end_location: Option, pub custom: U, pub node: T, } impl Located { - pub fn new(location: Location, node: T) -> Self { - Self { location, custom: (), node } + pub fn new(location: Location, end_location: Location, node: T) -> Self { + Self { location, end_location: Some(end_location), custom: (), node } } } \n diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs index 17900fe..e4fcc44 100644 --- a/ast/src/ast_gen.rs +++ b/ast/src/ast_gen.rs @@ -10,14 +10,16 @@ type Ident = String; #[derive(Debug, PartialEq)] pub struct Located { pub location: Location, + pub end_location: Option, pub custom: U, pub node: T, } impl Located { - pub fn new(location: Location, node: T) -> Self { + pub fn new(location: Location, end_location: Location, node: T) -> Self { Self { location, + end_location: Some(end_location), custom: (), node, } @@ -529,6 +531,7 @@ pub mod fold { Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, + end_location: node.end_location, node: f(folder, node.node)?, }) } diff --git a/ast/src/constant.rs b/ast/src/constant.rs index b73bab6..5de3b34 100644 --- a/ast/src/constant.rs +++ b/ast/src/constant.rs @@ -121,6 +121,7 @@ impl crate::fold::Fold for ConstantOptimizer { node: expr, custom: node.custom, location: node.location, + end_location: node.end_location, }) } _ => crate::fold::fold_expr(self, node), @@ -137,16 +138,19 @@ mod tests { use crate::fold::Fold; use crate::*; - let location = Location::new(0, 0); + let start = Default::default(); + let end = None; let custom = (); let ast = Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Tuple { ctx: ExprContext::Load, elts: vec![ Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: BigInt::from(1).into(), @@ -154,7 +158,8 @@ mod tests { }, }, Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: BigInt::from(2).into(), @@ -162,13 +167,15 @@ mod tests { }, }, Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Tuple { ctx: ExprContext::Load, elts: vec![ Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: BigInt::from(3).into(), @@ -176,7 +183,8 @@ mod tests { }, }, Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: BigInt::from(4).into(), @@ -184,7 +192,8 @@ mod tests { }, }, Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: BigInt::from(5).into(), @@ -203,7 +212,8 @@ mod tests { assert_eq!( new_ast, Located { - location, + location: start, + end_location: end, custom, node: ExprKind::Constant { value: Constant::Tuple(vec![ diff --git a/parser/python.lalrpop b/parser/python.lalrpop index 224450e..cf43f0f 100644 --- a/parser/python.lalrpop +++ b/parser/python.lalrpop @@ -66,9 +66,10 @@ SmallStatement: ast::Stmt = { }; PassStatement: ast::Stmt = { - "pass" => { + "pass" => { ast::Stmt { location, + end_location: Some(end_location), custom: (), node: ast::StmtKind::Pass, } @@ -76,9 +77,10 @@ PassStatement: ast::Stmt = { }; DelStatement: ast::Stmt = { - "del" => { + "del" => { ast::Stmt { location, + end_location: Some(end_location), custom: (), node: ast::StmtKind::Delete { targets }, } @@ -86,12 +88,13 @@ DelStatement: ast::Stmt = { }; ExpressionStatement: ast::Stmt = { - => { + => { // Just an expression, no assignment: if suffix.is_empty() { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Expr { value: Box::new(expression) } } } else { @@ -107,14 +110,16 @@ ExpressionStatement: ast::Stmt = { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Assign { targets, value, type_comment: None }, } } }, - => { + => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::AugAssign { target: Box::new(target), op, @@ -122,11 +127,12 @@ ExpressionStatement: ast::Stmt = { }, } }, - ":" => { + ":" => { let simple = matches!(target.node, ast::ExprKind::Name { .. }); ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::AnnAssign { target: Box::new(target), annotation: Box::new(annotation), @@ -183,31 +189,35 @@ AugAssign: ast::Operator = { }; FlowStatement: ast::Stmt = { - "break" => { + "break" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Break, } }, - "continue" => { + "continue" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Continue, } }, - "return" => { + "return" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Return { value: value.map(Box::new) }, } }, - => { + => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Expr { value: Box::new(expression) }, } }, @@ -215,35 +225,39 @@ FlowStatement: ast::Stmt = { }; RaiseStatement: ast::Stmt = { - "raise" => { + "raise" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Raise { exc: None, cause: None }, } }, - "raise" => { + "raise" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }, } }, }; ImportStatement: ast::Stmt = { - "import" >> => { + "import" >> => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Import { names }, } }, - "from" "import" => { + "from" "import" => { let (level, module) = source; ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::ImportFrom { level, module, @@ -268,18 +282,18 @@ ImportDots: usize = { }; ImportAsNames: Vec = { - >> => i, - "(" >> ","? ")" => i, - "*" => { + >> => i, + "(" >> ","? ")" => i, + "*" => { // Star import all - vec![ast::Alias::new(location, ast::AliasData { name: "*".to_string(), asname: None })] + vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })] }, }; #[inline] ImportAsAlias: ast::Alias = { - => ast::Alias::new(location, ast::AliasData { name, asname: a.map(|a| a.1) }), + => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }), } // A name like abc or abc.def.ghi @@ -296,30 +310,33 @@ DottedName: String = { }; GlobalStatement: ast::Stmt = { - "global" > => { + "global" > => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Global { names } } }, }; NonlocalStatement: ast::Stmt = { - "nonlocal" > => { + "nonlocal" > => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Nonlocal { names } } }, }; AssertStatement: ast::Stmt = { - "assert" => { + "assert" => { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Assert { test: Box::new(test), msg: msg.map(|e| Box::new(e.1)) @@ -339,7 +356,7 @@ CompoundStatement: ast::Stmt = { }; IfStatement: ast::Stmt = { - "if" ":" => { + "if" ":" => { // Determine last else: let mut last = s3.map(|s| s.2).unwrap_or_default(); @@ -348,6 +365,7 @@ IfStatement: ast::Stmt = { let x = ast::Stmt { custom: (), location: i.0, + end_location: Some(i.5), node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last }, }; last = vec![x]; @@ -356,17 +374,19 @@ IfStatement: ast::Stmt = { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::If { test: Box::new(test), body, orelse: last } } }, }; WhileStatement: ast::Stmt = { - "while" ":" => { + "while" ":" => { let orelse = s2.map(|s| s.2).unwrap_or_default(); ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::While { test: Box::new(test), body, @@ -377,7 +397,7 @@ WhileStatement: ast::Stmt = { }; ForStatement: ast::Stmt = { - "for" "in" ":" => { + "for" "in" ":" => { let orelse = s2.map(|s| s.2).unwrap_or_default(); let target = Box::new(target); let iter = Box::new(iter); @@ -387,17 +407,18 @@ ForStatement: ast::Stmt = { } else { ast::StmtKind::For { target, iter, body, orelse, type_comment } }; - ast::Stmt::new(location, node) + ast::Stmt::new(location, end_location, node) }, }; TryStatement: ast::Stmt = { - "try" ":" => { + "try" ":" => { let orelse = else_suite.map(|s| s.2).unwrap_or_default(); let finalbody = finally.map(|s| s.2).unwrap_or_default(); ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Try { body, handlers, @@ -406,13 +427,14 @@ TryStatement: ast::Stmt = { }, } }, - "try" ":" => { + "try" ":" => { let handlers = vec![]; let orelse = vec![]; let finalbody = finally.2; ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::Try { body, handlers, @@ -424,9 +446,10 @@ TryStatement: ast::Stmt = { }; ExceptClause: ast::Excepthandler = { - "except" ":" => { + "except" ":" => { ast::Excepthandler::new( location, + end_location, ast::ExcepthandlerKind::ExceptHandler { type_: typ.map(Box::new), name: None, @@ -434,9 +457,10 @@ ExceptClause: ast::Excepthandler = { }, ) }, - "except" ":" => { + "except" ":" => { ast::Excepthandler::new( location, + end_location, ast::ExcepthandlerKind::ExceptHandler { type_: Some(Box::new(x.0)), name: Some(x.2), @@ -447,14 +471,14 @@ ExceptClause: ast::Excepthandler = { }; WithStatement: ast::Stmt = { - "with" > ":" => { + "with" > ":" => { let type_comment = None; let node = if is_async.is_some() { ast::StmtKind::AsyncWith { items, body, type_comment } } else { ast::StmtKind::With { items, body, type_comment } }; - ast::Stmt::new(location, node) + ast::Stmt::new(location, end_location, node) }, }; @@ -467,7 +491,7 @@ WithItem: ast::Withitem = { }; FuncDef: ast::Stmt = { - "def" " Test)?> ":" => { + "def" " Test)?> ":" => { let args = Box::new(args); let returns = r.map(|x| Box::new(x.1)); let type_comment = None; @@ -476,7 +500,7 @@ FuncDef: ast::Stmt = { } else { ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment } }; - ast::Stmt::new(location, node) + ast::Stmt::new(location, end_location, node) }, }; @@ -574,16 +598,17 @@ ParameterDef: (ast::Arg, Option) = { }; UntypedParameter: ast::Arg = { - => ast::Arg::new( + => ast::Arg::new( location, + end_location, ast::ArgData { arg, annotation: None, type_comment: None }, ), }; TypedParameter: ast::Arg = { - => { + => { let annotation = a.map(|x| Box::new(x.1)); - ast::Arg::new(location, ast::ArgData { arg, annotation, type_comment: None }) + ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None }) }, }; @@ -620,7 +645,7 @@ KwargParameter: Option> = { }; ClassDef: ast::Stmt = { - "class" ":" => { + "class" ":" => { let (bases, keywords) = match a { Some((_, arg, _)) => (arg.args, arg.keywords), None => (vec![], vec![]), @@ -628,6 +653,7 @@ ClassDef: ast::Stmt = { ast::Stmt { custom: (), location, + end_location: Some(end_location), node: ast::StmtKind::ClassDef { name, bases, @@ -641,29 +667,32 @@ ClassDef: ast::Stmt = { // Decorators: Decorator: ast::Expr = { - "@" "\n" => { + "@" "\n" => { p }, }; YieldExpr: ast::Expr = { - "yield" => ast::Expr { + "yield" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Yield { value: value.map(Box::new) } }, - "yield" "from" => ast::Expr { + "yield" "from" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::YieldFrom { value: Box::new(e) } }, }; Test: ast::Expr = { - => { + => { if let Some(c) = condition { ast::Expr { location: c.0, + end_location: Some(c.5), custom: (), node: ast::ExprKind::IfExp { test: Box::new(c.2), @@ -679,14 +708,16 @@ Test: ast::Expr = { }; NamedExpressionTest: ast::Expr = { - ?> ":" => { + "lambda" ?> ":" => { let p = p.unwrap_or_else(|| { ast::Arguments { posonlyargs: vec![], @@ -713,6 +744,7 @@ LambdaDef: ast::Expr = { }); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Lambda { args: Box::new(p), @@ -723,7 +755,7 @@ LambdaDef: ast::Expr = { } OrTest: ast::Expr = { - => { + => { if e2.is_empty() { e1 } else { @@ -731,6 +763,7 @@ OrTest: ast::Expr = { values.extend(e2.into_iter().map(|e| e.1)); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values } } @@ -739,7 +772,7 @@ OrTest: ast::Expr = { }; AndTest: ast::Expr = { - => { + => { if e2.is_empty() { e1 } else { @@ -747,6 +780,7 @@ AndTest: ast::Expr = { values.extend(e2.into_iter().map(|e| e.1)); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values } } @@ -755,8 +789,9 @@ AndTest: ast::Expr = { }; NotTest: ast::Expr = { - "not" => ast::Expr { + "not" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not } }, @@ -764,10 +799,11 @@ NotTest: ast::Expr = { }; Comparison: ast::Expr = { - => { + => { let (ops, comparators) = comparisons.into_iter().unzip(); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators } } @@ -789,8 +825,9 @@ CompOp: ast::Cmpop = { }; Expression: ast::Expr = { - "|" => ast::Expr { + "|" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) } }, @@ -798,8 +835,9 @@ Expression: ast::Expr = { }; XorExpression: ast::Expr = { - "^" => ast::Expr { + "^" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) } }, @@ -807,8 +845,9 @@ XorExpression: ast::Expr = { }; AndExpression: ast::Expr = { - "&" => ast::Expr { + "&" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) } }, @@ -816,8 +855,9 @@ AndExpression: ast::Expr = { }; ShiftExpression: ast::Expr = { - => ast::Expr { + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) } }, @@ -830,8 +870,9 @@ ShiftOp: ast::Operator = { }; ArithmeticExpression: ast::Expr = { - => ast::Expr { + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } }, @@ -844,8 +885,9 @@ AddOp: ast::Operator = { }; Term: ast::Expr = { - => ast::Expr { + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } }, @@ -861,8 +903,9 @@ MulOp: ast::Operator = { }; Factor: ast::Expr = { - => ast::Expr { + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } }, @@ -876,11 +919,12 @@ UnaryOp: ast::Unaryop = { }; Power: ast::Expr = { - => { + => { match e2 { None => e, - Some((location, _, b)) => ast::Expr { + Some((location, _, b, end_location)) => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) } }, @@ -889,10 +933,11 @@ Power: ast::Expr = { }; AtomExpr: ast::Expr = { - => { + => { if is_await.is_some() { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Await { value: Box::new(atom) } } @@ -904,27 +949,30 @@ AtomExpr: ast::Expr = { AtomExpr2: ast::Expr = { Atom, - "(" ")" => { + "(" ")" => { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords } } }, - "[" "]" => ast::Expr { + "[" "]" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load } }, - "." => ast::Expr { + "." => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load } }, }; SubscriptList: ast::Expr = { - => { + => { if s2.is_empty() && trailing_comma.is_none() { s1 } else { @@ -935,6 +983,7 @@ SubscriptList: ast::Expr = { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load }, } @@ -944,12 +993,13 @@ SubscriptList: ast::Expr = { Subscript: ast::Expr = { Test, - ":" => { + ":" => { let lower = e1.map(Box::new); let upper = e2.map(Box::new); let step = e3.flatten().map(Box::new); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Slice { lower, upper, step } } @@ -961,40 +1011,44 @@ SliceOp: Option = { } Atom: ast::Expr = { - =>? parse_strings(s).map_err(|e| e.into()), - => ast::Expr { + =>? parse_strings(s).map_err(|e| e.into()), + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Constant { value, kind: None } }, - => ast::Expr { + => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load } }, - "[" "]" => { + "[" "]" => { let elts = e.unwrap_or_default(); ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load } } }, - "[" "]" => { + "[" "]" => { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::ListComp { elt: Box::new(elt), generators } } }, - "(" ")" =>? { + "(" ")" =>? { match elements { Some(elt) => { match elt.node { ast::ExprKind::Starred { .. } => { Err(LexicalError{ error : LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location, + location: location, }.into()) }, _ => { @@ -1003,29 +1057,30 @@ Atom: ast::Expr = { } }, None => { - Ok(ast::Expr { + Ok(ast::Expr::new( location, - custom: (), - node: ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } - }) + end_location, + ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } + )) } } }, "(" ")" => e, - "(" ")" => { + "(" ")" => { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators } } }, - "(" "**" ")" =>? { + "(" "**" ")" =>? { Err(LexicalError{ error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), - location, + location: location, }.into()) }, - "{" "}" => { + "{" "}" => { let pairs = e.unwrap_or_default(); let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) { @@ -1035,9 +1090,11 @@ Atom: ast::Expr = { fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr { let location = items[0].0.location; + let end_location = items[0].0.end_location; let (keys, values) = items.drain(..).unzip(); ast::Expr { location, + end_location, custom: (), node: ast::ExprKind::Dict { keys, values } } @@ -1064,13 +1121,15 @@ Atom: ast::Expr = { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Dict { keys, values } } }, - "{" "}" => { + "{" "}" => { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::DictComp { key: Box::new(e1.0), @@ -1079,22 +1138,24 @@ Atom: ast::Expr = { } } }, - "{" "}" => ast::Expr { + "{" "}" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Set { elts } }, - "{" "}" => { + "{" "}" => { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::SetComp { elt: Box::new(elt), generators } } }, - "True" => ast::Expr::new(location, ast::ExprKind::Constant { value: true.into(), kind: None }), - "False" => ast::Expr::new(location, ast::ExprKind::Constant { value: false.into(), kind: None }), - "None" => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }), - "..." => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }), + "True" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }), + "False" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }), + "None" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }), + "..." => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }), }; ListLiteralValues: Vec = { @@ -1141,12 +1202,13 @@ TestList: ast::Expr = { }; GenericList: ast::Expr = { - > => { + > => { if elts.len() == 1 && trailing_comma.is_none() { elts.into_iter().next().unwrap() } else { ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load } } @@ -1156,8 +1218,9 @@ GenericList: ast::Expr = { // Test StarExpr: ast::Expr = { - "*" => ast::Expr { + "*" => ast::Expr { location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, } @@ -1167,7 +1230,7 @@ StarExpr: ast::Expr = { CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { + "for" "in" => { let is_async = is_async.is_some(); ast::Comprehension { target: Box::new(target), @@ -1188,11 +1251,12 @@ ArgumentList: ArgumentList = { } }; -FunctionArgument: (Option<(ast::Location, Option)>, ast::Expr) = { +FunctionArgument: (Option<(ast::Location, ast::Location, Option)>, ast::Expr) = { => { let expr = match c { Some(c) => ast::Expr { location: e.location, + end_location: e.end_location, custom: (), node: ast::ExprKind::GeneratorExp { elt: Box::new(e), @@ -1203,15 +1267,16 @@ FunctionArgument: (Option<(ast::Location, Option)>, ast::Expr) = { }; (None, expr) }, - "=" => (Some((location, Some(i))), e), - "*" => { + "=" => (Some((location, end_location, Some(i))), e), + "*" => { let expr = ast::Expr::new( location, + end_location, ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, ); (None, expr) }, - "**" => (Some((location, None)), e), + "**" => (Some((location, end_location, None)), e), }; #[inline] diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 342e47d..2326583 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -7,17 +7,18 @@ use crate::{ use std::{iter, mem, str}; struct FStringParser { - str_location: Location, + str_start: Location, + str_end: Location, } impl FStringParser { - fn new(str_location: Location) -> Self { - Self { str_location } + fn new(str_start: Location, str_end: Location) -> Self { + Self { str_start, str_end } } #[inline] fn expr(&self, node: ExprKind) -> Expr { - Expr::new(self.str_location, node) + Expr::new(self.str_start, self.str_end, node) } fn parse_formatted_value<'a>( @@ -305,11 +306,18 @@ fn parse_fstring_expr(source: &str) -> Result { /// Parse an fstring from a string, located at a certain position in the sourcecode. /// In case of errors, we will get the location and the error returned. -pub fn parse_located_fstring(source: &str, location: Location) -> Result, FStringError> { - FStringParser::new(location) +pub fn parse_located_fstring( + source: &str, + start: Location, + end: Location, +) -> Result, FStringError> { + FStringParser::new(start, end) .parse(source.chars().peekable(), 0) .map(|(e, _)| e) - .map_err(|error| FStringError { error, location }) + .map_err(|error| FStringError { + error, + location: start, + }) } #[cfg(test)] @@ -317,7 +325,7 @@ mod tests { use super::*; fn parse_fstring(source: &str) -> Result, FStringErrorType> { - FStringParser::new(Location::default()) + FStringParser::new(Location::default(), Location::default()) .parse(source.chars().peekable(), 0) .map(|(e, _)| e) } diff --git a/parser/src/function.rs b/parser/src/function.rs index 2a384dd..28cb44f 100644 --- a/parser/src/function.rs +++ b/parser/src/function.rs @@ -45,7 +45,10 @@ pub fn parse_params( Ok((posonly, names, defaults)) } -type FunctionArgument = (Option<(ast::Location, Option)>, ast::Expr); +type FunctionArgument = ( + Option<(ast::Location, ast::Location, Option)>, + ast::Expr, +); pub fn parse_args(func_args: Vec) -> Result { let mut args = vec![]; @@ -54,12 +57,12 @@ pub fn parse_args(func_args: Vec) -> Result { + Some((start, end, name)) => { if let Some(keyword_name) = &name { if keyword_names.contains(keyword_name) { return Err(LexicalError { error: LexicalErrorType::DuplicateKeywordArgumentError, - location, + location: start, }); } @@ -67,7 +70,8 @@ pub fn parse_args(func_args: Vec) -> Result Result Result\").unwrap()" row: 1, column: 1, }, + end_location: Some( + Location { + row: 5, + column: 7, + }, + ), custom: (), node: ClassDef { name: "Foo", @@ -17,6 +23,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 1, column: 11, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Name { id: "A", @@ -28,6 +40,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 1, column: 14, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: Name { id: "B", @@ -42,6 +60,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 2, column: 2, }, + end_location: Some( + Location { + row: 4, + column: 2, + }, + ), custom: (), node: FunctionDef { name: "__init__", @@ -53,6 +77,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 2, column: 15, }, + end_location: Some( + Location { + row: 2, + column: 19, + }, + ), custom: (), node: ArgData { arg: "self", @@ -73,6 +103,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 3, column: 3, }, + end_location: Some( + Location { + row: 3, + column: 7, + }, + ), custom: (), node: Pass, }, @@ -87,6 +123,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 4, column: 2, }, + end_location: Some( + Location { + row: 5, + column: 7, + }, + ), custom: (), node: FunctionDef { name: "method_with_default", @@ -98,6 +140,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 4, column: 26, }, + end_location: Some( + Location { + row: 4, + column: 30, + }, + ), custom: (), node: ArgData { arg: "self", @@ -110,6 +158,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 4, column: 32, }, + end_location: Some( + Location { + row: 4, + column: 35, + }, + ), custom: (), node: ArgData { arg: "arg", @@ -128,6 +182,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 4, column: 36, }, + end_location: Some( + Location { + row: 4, + column: 45, + }, + ), custom: (), node: Constant { value: Str( @@ -144,6 +204,12 @@ expression: "parse_program(source, \"\").unwrap()" row: 5, column: 3, }, + end_location: Some( + Location { + row: 5, + column: 7, + }, + ), custom: (), node: Pass, }, diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap index f7f9b52..a75ccdc 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), custom: (), node: DictComp { key: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 4, + }, + ), custom: (), node: Name { id: "x1", @@ -25,6 +37,12 @@ Located { row: 1, column: 6, }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), custom: (), node: Name { id: "x2", @@ -38,6 +56,12 @@ Located { row: 1, column: 13, }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), custom: (), node: Name { id: "y", @@ -49,6 +73,12 @@ Located { row: 1, column: 18, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: Name { id: "z", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap index 52d4b6c..3018e3f 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 49, + }, + ), custom: (), node: ListComp { elt: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 3, + }, + ), custom: (), node: Name { id: "x", @@ -27,6 +39,12 @@ Located { row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), custom: (), node: Tuple { elts: [ @@ -35,6 +53,12 @@ Located { row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Name { id: "y", @@ -46,6 +70,12 @@ Located { row: 1, column: 11, }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), custom: (), node: Name { id: "y2", @@ -61,6 +91,12 @@ Located { row: 1, column: 17, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Name { id: "z", @@ -76,6 +112,12 @@ Located { row: 1, column: 23, }, + end_location: Some( + Location { + row: 1, + column: 24, + }, + ), custom: (), node: Name { id: "a", @@ -87,6 +129,12 @@ Located { row: 1, column: 28, }, + end_location: Some( + Location { + row: 1, + column: 29, + }, + ), custom: (), node: Name { id: "b", @@ -99,6 +147,12 @@ Located { row: 1, column: 35, }, + end_location: Some( + Location { + row: 1, + column: 38, + }, + ), custom: (), node: Compare { left: Located { @@ -106,6 +160,12 @@ Located { row: 1, column: 33, }, + end_location: Some( + Location { + row: 1, + column: 34, + }, + ), custom: (), node: Name { id: "a", @@ -121,6 +181,12 @@ Located { row: 1, column: 37, }, + end_location: Some( + Location { + row: 1, + column: 38, + }, + ), custom: (), node: Constant { value: Int( @@ -137,6 +203,12 @@ Located { row: 1, column: 44, }, + end_location: Some( + Location { + row: 1, + column: 48, + }, + ), custom: (), node: Compare { left: Located { @@ -144,6 +216,12 @@ Located { row: 1, column: 42, }, + end_location: Some( + Location { + row: 1, + column: 43, + }, + ), custom: (), node: Name { id: "a", @@ -159,6 +237,12 @@ Located { row: 1, column: 46, }, + end_location: Some( + Location { + row: 1, + column: 48, + }, + ), custom: (), node: Constant { value: Int( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap index 97b49bd..13c2fc6 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap index 0e03788..0b85a1a 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: GeneratorExp { elt: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 3, + }, + ), custom: (), node: Name { id: "x", @@ -27,6 +39,12 @@ Located { row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Name { id: "y", @@ -38,6 +56,12 @@ Located { row: 1, column: 13, }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), custom: (), node: Name { id: "z", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap index 4b34c0e..c628cf9 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap @@ -1,7 +1,6 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast - --- [ Located { @@ -9,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 3, + column: 9, + }, + ), custom: (), node: If { test: Located { @@ -16,6 +21,12 @@ expression: parse_ast row: 1, column: 4, }, + end_location: Some( + Location { + row: 1, + column: 5, + }, + ), custom: (), node: Constant { value: Int( @@ -30,6 +41,12 @@ expression: parse_ast row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Expr { value: Located { @@ -37,6 +54,12 @@ expression: parse_ast row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Int( @@ -54,6 +77,12 @@ expression: parse_ast row: 2, column: 1, }, + end_location: Some( + Location { + row: 3, + column: 1, + }, + ), custom: (), node: If { test: Located { @@ -61,6 +90,12 @@ expression: parse_ast row: 2, column: 6, }, + end_location: Some( + Location { + row: 2, + column: 7, + }, + ), custom: (), node: Constant { value: Int( @@ -75,6 +110,12 @@ expression: parse_ast row: 2, column: 9, }, + end_location: Some( + Location { + row: 2, + column: 11, + }, + ), custom: (), node: Expr { value: Located { @@ -82,6 +123,12 @@ expression: parse_ast row: 2, column: 9, }, + end_location: Some( + Location { + row: 2, + column: 11, + }, + ), custom: (), node: Constant { value: Int( @@ -99,6 +146,12 @@ expression: parse_ast row: 3, column: 7, }, + end_location: Some( + Location { + row: 3, + column: 9, + }, + ), custom: (), node: Expr { value: Located { @@ -106,6 +159,12 @@ expression: parse_ast row: 3, column: 7, }, + end_location: Some( + Location { + row: 3, + column: 9, + }, + ), custom: (), node: Constant { value: Int( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap index 782df39..4505d92 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 27, + }, + ), custom: (), node: GeneratorExp { elt: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 4, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: IfExp { test: Located { @@ -21,6 +33,12 @@ Located { row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), custom: (), node: Name { id: "y", @@ -32,6 +50,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 3, + }, + ), custom: (), node: Name { id: "x", @@ -43,6 +67,12 @@ Located { row: 1, column: 14, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: Name { id: "y", @@ -58,6 +88,12 @@ Located { row: 1, column: 20, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Name { id: "y", @@ -69,6 +105,12 @@ Located { row: 1, column: 25, }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), custom: (), node: Name { id: "z", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap index 1b8a545..61d45da 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap @@ -8,13 +8,25 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 33, + }, + ), custom: (), node: Expr { value: Located { location: Location { row: 1, - column: 8, + column: 1, }, + end_location: Some( + Location { + row: 1, + column: 33, + }, + ), custom: (), node: Call { func: Located { @@ -22,6 +34,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), custom: (), node: Name { id: "my_func", @@ -34,6 +52,12 @@ expression: parse_ast row: 1, column: 9, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Constant { value: Str( @@ -49,6 +73,12 @@ expression: parse_ast row: 1, column: 23, }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), custom: (), node: KeywordData { arg: Some( @@ -59,6 +89,12 @@ expression: parse_ast row: 1, column: 31, }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), custom: (), node: Constant { value: Int( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap index 2f70292..7e0395a 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- [ @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: Lambda { args: Arguments { @@ -25,6 +37,12 @@ expression: parse_ast row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: ArgData { arg: "x", @@ -37,6 +55,12 @@ expression: parse_ast row: 1, column: 11, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: ArgData { arg: "y", @@ -56,6 +80,12 @@ expression: parse_ast row: 1, column: 16, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: BinOp { left: Located { @@ -63,6 +93,12 @@ expression: parse_ast row: 1, column: 14, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: Name { id: "x", @@ -75,6 +111,12 @@ expression: parse_ast row: 1, column: 18, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: Name { id: "y", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap index 8a6d8eb..d691665 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 15, + }, + ), custom: (), node: ListComp { elt: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 3, + }, + ), custom: (), node: Name { id: "x", @@ -27,6 +39,12 @@ Located { row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Name { id: "y", @@ -38,6 +56,12 @@ Located { row: 1, column: 13, }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), custom: (), node: Name { id: "z", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap index 960151b..660f0fa 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap @@ -1,5 +1,5 @@ --- -source: parser/src/parser.rs +source: compiler/parser/src/parser.rs expression: parse_ast --- Located { @@ -7,6 +7,12 @@ Located { row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 24, + }, + ), custom: (), node: GeneratorExp { elt: Located { @@ -14,6 +20,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: NamedExpr { target: Located { @@ -21,6 +33,12 @@ Located { row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Name { id: "x", @@ -32,6 +50,12 @@ Located { row: 1, column: 9, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: BinOp { left: Located { @@ -39,6 +63,12 @@ Located { row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), custom: (), node: Name { id: "y", @@ -51,6 +81,12 @@ Located { row: 1, column: 11, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Constant { value: Int( @@ -70,6 +106,12 @@ Located { row: 1, column: 17, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Name { id: "y", @@ -81,6 +123,12 @@ Located { row: 1, column: 22, }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), custom: (), node: Name { id: "z", diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap index 3f828de..fcb2eec 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap @@ -8,13 +8,25 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 24, + }, + ), custom: (), node: Expr { value: Located { location: Location { row: 1, - column: 6, + column: 1, }, + end_location: Some( + Location { + row: 1, + column: 24, + }, + ), custom: (), node: Call { func: Located { @@ -22,6 +34,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 6, + }, + ), custom: (), node: Name { id: "print", @@ -34,6 +52,12 @@ expression: parse_ast row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), custom: (), node: Constant { value: Str( @@ -47,6 +71,12 @@ expression: parse_ast row: 1, column: 22, }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), custom: (), node: Constant { value: Int( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap index aae345a..49efd47 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap @@ -8,13 +8,25 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Expr { value: Located { location: Location { row: 1, - column: 6, + column: 1, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Call { func: Located { @@ -22,6 +34,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 6, + }, + ), custom: (), node: Name { id: "print", @@ -34,6 +52,12 @@ expression: parse_ast row: 1, column: 7, }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap index efbf32f..318a0f9 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap index 26243f5..d50b57d 100644 --- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap @@ -1,7 +1,6 @@ --- -source: parser/src/parser.rs -expression: parse_program(&source).unwrap() - +source: compiler/parser/src/parser.rs +expression: "parse_program(source, \"\").unwrap()" --- [ Located { @@ -9,6 +8,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Assign { targets: [ @@ -17,6 +22,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 5, + }, + ), custom: (), node: Tuple { elts: [ @@ -25,6 +36,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 2, + }, + ), custom: (), node: Name { id: "a", @@ -36,6 +53,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 4, }, + end_location: Some( + Location { + row: 1, + column: 5, + }, + ), custom: (), node: Name { id: "b", @@ -52,6 +75,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Tuple { elts: [ @@ -60,6 +89,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 8, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Int( @@ -73,6 +108,12 @@ expression: parse_program(&source).unwrap() row: 1, column: 11, }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), custom: (), node: Constant { value: Int( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap index a486e11..3cce5b1 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap index a486e11..3cce5b1 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap index ccddd2a..2ab5692 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Str( @@ -36,6 +54,12 @@ expression: parse_ast row: 1, column: 10, }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), custom: (), node: FormattedValue { value: Located { @@ -43,6 +67,12 @@ expression: parse_ast row: 1, column: 2, }, + end_location: Some( + Location { + row: 1, + column: 5, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap index 380a1fc..fd88e7b 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap index 66db231..a08dde8 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 21, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap index dd6d338..ba2ae38 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap index b5d6f94..7d21fea 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), custom: (), node: JoinedStr { values: [ @@ -23,6 +35,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap index 380a1fc..9c93348 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap index 74bf59c..0af6e48 100644 --- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap +++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap @@ -8,6 +8,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 18, + }, + ), custom: (), node: Expr { value: Located { @@ -15,6 +21,12 @@ expression: parse_ast row: 1, column: 1, }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), custom: (), node: Constant { value: Str( diff --git a/parser/src/string.rs b/parser/src/string.rs index 4c8ff85..078ac1e 100644 --- a/parser/src/string.rs +++ b/parser/src/string.rs @@ -6,9 +6,12 @@ use crate::{ }; use itertools::Itertools; -pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result { +pub fn parse_strings( + values: Vec<(Location, (String, StringKind), Location)>, +) -> Result { // Preserve the initial location and kind. - let initial_location = values[0].0; + let initial_start = values[0].0; + let initial_end = values[0].2; let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned()); // Determine whether the list of values contains any f-strings. (If not, we can return a @@ -21,7 +24,8 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result| -> Expr { Expr::new( - initial_location, + initial_start, + initial_end, ExprKind::Constant { value: Constant::Str(current.drain(..).join("")), kind: initial_kind.clone(), @@ -29,15 +33,17 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result current.push(string), StringKind::F => { has_fstring = true; - for value in parse_located_fstring(&string, location).map_err(|e| LexicalError { - location, - error: LexicalErrorType::FStringError(e.error), - })? { + for value in + parse_located_fstring(&string, start, end).map_err(|e| LexicalError { + location: start, + error: LexicalErrorType::FStringError(e.error), + })? + { match value.node { ExprKind::FormattedValue { .. } => { if !current.is_empty() { @@ -63,7 +69,11 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result