From 03ccd4b9d89a642fcf055dcb8f13f8dd371f9968 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Aug 2022 06:48:47 +0900 Subject: [PATCH] Add source_path to ParseError --- codegen/src/compile.rs | 2 +- parser/src/error.rs | 14 +++++++++++-- parser/src/fstring.rs | 2 +- parser/src/lib.rs | 2 +- parser/src/parser.rs | 46 +++++++++++++++++++++--------------------- parser/src/string.rs | 16 +++++++-------- src/lib.rs | 22 +++++++++----------- 7 files changed, 55 insertions(+), 49 deletions(-) diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs index f7dd272..59d9faa 100644 --- a/codegen/src/compile.rs +++ b/codegen/src/compile.rs @@ -2707,7 +2707,7 @@ mod tests { "source_path".to_owned(), "".to_owned(), ); - let ast = parser::parse_program(source).unwrap(); + let ast = parser::parse_program(source, "").unwrap(); let symbol_scope = SymbolTable::scan_program(&ast).unwrap(); compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() diff --git a/parser/src/error.rs b/parser/src/error.rs index cc53ef2..d256e2a 100644 --- a/parser/src/error.rs +++ b/parser/src/error.rs @@ -124,6 +124,7 @@ impl From for LalrpopError { pub struct ParseError { pub error: ParseErrorType, pub location: Location, + pub source_path: String, } #[derive(Debug, PartialEq)] @@ -141,21 +142,28 @@ pub enum ParseErrorType { } /// Convert `lalrpop_util::ParseError` to our internal type -impl From> for ParseError { - fn from(err: LalrpopError) -> Self { +impl ParseError { + pub(crate) fn from_lalrpop( + err: LalrpopError, + source_path: &str, + ) -> Self { + let source_path = source_path.to_owned(); match err { // TODO: Are there cases where this isn't an EOF? LalrpopError::InvalidToken { location } => ParseError { error: ParseErrorType::Eof, location, + source_path, }, LalrpopError::ExtraToken { token } => ParseError { error: ParseErrorType::ExtraToken(token.1), location: token.0, + source_path, }, LalrpopError::User { error } => ParseError { error: ParseErrorType::Lexical(error.error), location: error.location, + source_path, }, LalrpopError::UnrecognizedToken { token, expected } => { // Hacky, but it's how CPython does it. See PyParser_AddToken, @@ -164,11 +172,13 @@ impl From> for ParseError { ParseError { error: ParseErrorType::UnrecognizedToken(token.1, expected), location: token.0, + source_path, } } LalrpopError::UnrecognizedEOF { location, .. } => ParseError { error: ParseErrorType::Eof, location, + source_path, }, } } diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 2869306..11d75a0 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> { fn parse_fstring_expr(source: &str) -> Result { let fstring_body = format!("({})", source); - parse_expression(&fstring_body) + parse_expression(&fstring_body, "") } /// Parse an fstring from a string, located at a certain position in the sourcecode. diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 694e0a5..f65f736 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -11,7 +11,7 @@ //! use rustpython_parser::{parser, ast}; //! //! let python_source = "print('Hello world')"; -//! let python_ast = parser::parse_expression(python_source).unwrap(); +//! let python_ast = parser::parse_expression(python_source, "").unwrap(); //! //! ``` diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 990f846..3199701 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -20,8 +20,8 @@ use crate::python; */ /// Parse a full python program, containing usually multiple lines. -pub fn parse_program(source: &str) -> Result { - parse(source, Mode::Module).map(|top| match top { +pub fn parse_program(source: &str, source_path: &str) -> Result { + parse(source, Mode::Module, source_path).map(|top| match top { ast::Mod::Module { body, .. } => body, _ => unreachable!(), }) @@ -33,7 +33,7 @@ pub fn parse_program(source: &str) -> Result { /// ``` /// extern crate num_bigint; /// use rustpython_parser::{parser, ast}; -/// let expr = parser::parse_expression("1 + 2").unwrap(); +/// let expr = parser::parse_expression("1 + 2", "").unwrap(); /// /// assert_eq!( /// expr, @@ -63,22 +63,22 @@ pub fn parse_program(source: &str) -> Result { /// ); /// /// ``` -pub fn parse_expression(source: &str) -> Result { - parse(source, Mode::Expression).map(|top| match top { +pub fn parse_expression(source: &str, path: &str) -> Result { + parse(source, Mode::Expression, path).map(|top| match top { ast::Mod::Expression { body } => *body, _ => unreachable!(), }) } // Parse a given source code -pub fn parse(source: &str, mode: Mode) -> Result { +pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result { let lxr = lexer::make_tokenizer(source); let marker_token = (Default::default(), mode.to_marker(), Default::default()); let tokenizer = iter::once(Ok(marker_token)).chain(lxr); python::TopParser::new() .parse(tokenizer) - .map_err(ParseError::from) + .map_err(|e| ParseError::from_lalrpop(e, source_path)) } #[cfg(test)] @@ -87,56 +87,56 @@ mod tests { #[test] fn test_parse_empty() { - let parse_ast = parse_program("").unwrap(); + let parse_ast = parse_program("", "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_string() { let source = String::from("'Hello world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string() { let source = String::from("f'Hello world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_hello() { let source = String::from("print('Hello world')"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_2() { let source = String::from("print('Hello world', 2)"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_kwargs() { let source = String::from("my_func('positional', keyword=2)"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_elif_else() { let source = String::from("if 1: 10\nelif 2: 20\nelse: 30"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_lambda() { let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; - let parse_ast = parse_program(source).unwrap(); + let parse_ast = parse_program(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -144,7 +144,7 @@ mod tests { fn test_parse_tuples() { let source = "a, b = 4, 5"; - insta::assert_debug_snapshot!(parse_program(source).unwrap()); + insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); } #[test] @@ -155,48 +155,48 @@ class Foo(A, B): pass def method_with_default(self, arg='default'): pass"; - insta::assert_debug_snapshot!(parse_program(source).unwrap()); + insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); } #[test] fn test_parse_dict_comprehension() { let source = String::from("{x1: x2 for y in z}"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_list_comprehension() { let source = String::from("[x for y in z]"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_double_list_comprehension() { let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_generator_comprehension() { let source = String::from("(x for y in z)"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_named_expression_generator_comprehension() { let source = String::from("(x := y + 1 for y in z)"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_else_generator_comprehension() { let source = String::from("(x if y else y for y in z)"); - let parse_ast = parse_expression(&source).unwrap(); + let parse_ast = parse_expression(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } } diff --git a/parser/src/string.rs b/parser/src/string.rs index 02a5514..1c16f7f 100644 --- a/parser/src/string.rs +++ b/parser/src/string.rs @@ -88,56 +88,56 @@ mod tests { #[test] fn test_parse_string_concat() { let source = String::from("'Hello ' 'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_1() { let source = String::from("'Hello ' u'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_2() { let source = String::from("u'Hello ' 'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_1() { let source = String::from("'Hello ' f'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_2() { let source = String::from("'Hello ' f'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_3() { let source = String::from("'Hello ' f'world{\"!\"}'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_1() { let source = String::from("u'Hello ' f'world'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_2() { let source = String::from("u'Hello ' f'world' '!'"); - let parse_ast = parse_program(&source).unwrap(); + let parse_ast = parse_program(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } } diff --git a/src/lib.rs b/src/lib.rs index 9e0a5c2..6996589 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,22 +43,18 @@ impl fmt::Display for CompileError { impl CompileError { fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self { - CompileError { + Self { error: error.error.into(), location: error.location, source_path: error.source_path, statement: get_statement(source, error.location), } } - fn from_parse( - error: rustpython_parser::error::ParseError, - source: &str, - source_path: String, - ) -> Self { - CompileError { + fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self { + Self { error: error.error.into(), location: error.location, - source_path, + source_path: error.source_path, statement: get_statement(source, error.location), } } @@ -83,9 +79,9 @@ pub fn compile( compile::Mode::Eval => parser::Mode::Expression, compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive, }; - let mut ast = match parser::parse(source, parser_mode) { + let mut ast = match parser::parse(source, parser_mode, &source_path) { Ok(x) => x, - Err(e) => return Err(CompileError::from_parse(e, source, source_path)), + Err(e) => return Err(CompileError::from_parse(e, source)), }; if opts.optimize > 0 { ast = ConstantOptimizer::new() @@ -101,14 +97,14 @@ pub fn compile_symtable( mode: compile::Mode, source_path: &str, ) -> Result { - let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned()); + let parse_err = |e| CompileError::from_parse(e, source); let res = match mode { compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => { - let ast = parser::parse_program(source).map_err(parse_err)?; + let ast = parser::parse_program(source, source_path).map_err(parse_err)?; symboltable::SymbolTable::scan_program(&ast) } compile::Mode::Eval => { - let expr = parser::parse_expression(source).map_err(parse_err)?; + let expr = parser::parse_expression(source, source_path).map_err(parse_err)?; symboltable::SymbolTable::scan_expr(&expr) } };