mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-12 15:45:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add source_path to ParseError

Browse source
This commit is contained in:
Jeong YunWon 2022-08-22 06:48:47 +09:00
parent cf10e800a6
commit 03ccd4b9d8
7 changed files with 55 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

2
codegen/src/compile.rs
View file

@ -2707,7 +2707,7 @@ mod tests {
"source_path".to_owned(),
"<module>".to_owned(),
);
let ast = parser::parse_program(source).unwrap();
let ast = parser::parse_program(source, "<test>").unwrap();
let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
compiler.compile_program(&ast, symbol_scope).unwrap();
compiler.pop_code_object()

14
parser/src/error.rs
View file

@ -124,6 +124,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
pub struct ParseError {
pub error: ParseErrorType,
pub location: Location,
pub source_path: String,
}
#[derive(Debug, PartialEq)]
@ -141,21 +142,28 @@ pub enum ParseErrorType {
}
/// Convert `lalrpop_util::ParseError` to our internal type
impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
impl ParseError {
pub(crate) fn from_lalrpop(
err: LalrpopError<Location, Tok, LexicalError>,
source_path: &str,
) -> Self {
let source_path = source_path.to_owned();
match err {
// TODO: Are there cases where this isn't an EOF?
LalrpopError::InvalidToken { location } => ParseError {
error: ParseErrorType::Eof,
location,
source_path,
},
LalrpopError::ExtraToken { token } => ParseError {
error: ParseErrorType::ExtraToken(token.1),
location: token.0,
source_path,
},
LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error),
location: error.location,
source_path,
},
LalrpopError::UnrecognizedToken { token, expected } => {
// Hacky, but it's how CPython does it. See PyParser_AddToken,
@ -164,11 +172,13 @@ impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
ParseError {
error: ParseErrorType::UnrecognizedToken(token.1, expected),
location: token.0,
source_path,
}
}
LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
error: ParseErrorType::Eof,
location,
source_path,
},
}
}

2
parser/src/fstring.rs
View file

@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> {
fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
let fstring_body = format!("({})", source);
parse_expression(&fstring_body)
parse_expression(&fstring_body, "<fstring>")
}
/// Parse an fstring from a string, located at a certain position in the sourcecode.

2
parser/src/lib.rs
View file

@ -11,7 +11,7 @@
//! use rustpython_parser::{parser, ast};
//!
//! let python_source = "print('Hello world')";
//! let python_ast = parser::parse_expression(python_source).unwrap();
//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
//!
//! ```

46
parser/src/parser.rs
View file

@ -20,8 +20,8 @@ use crate::python;
*/
/// Parse a full python program, containing usually multiple lines.
pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
parse(source, Mode::Module).map(|top| match top {
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
parse(source, Mode::Module, source_path).map(|top| match top {
ast::Mod::Module { body, .. } => body,
_ => unreachable!(),
})
@ -33,7 +33,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
/// ```
/// extern crate num_bigint;
/// use rustpython_parser::{parser, ast};
/// let expr = parser::parse_expression("1 + 2").unwrap();
/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
///
/// assert_eq!(
/// expr,
@ -63,22 +63,22 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
/// );
///
/// ```
pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
parse(source, Mode::Expression).map(|top| match top {
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
parse(source, Mode::Expression, path).map(|top| match top {
ast::Mod::Expression { body } => *body,
_ => unreachable!(),
})
}
// Parse a given source code
pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
let lxr = lexer::make_tokenizer(source);
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
python::TopParser::new()
.parse(tokenizer)
.map_err(ParseError::from)
.map_err(|e| ParseError::from_lalrpop(e, source_path))
}
#[cfg(test)]
@ -87,56 +87,56 @@ mod tests {
#[test]
fn test_parse_empty() {
let parse_ast = parse_program("").unwrap();
let parse_ast = parse_program("", "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string() {
let source = String::from("'Hello world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string() {
let source = String::from("f'Hello world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_hello() {
let source = String::from("print('Hello world')");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_2() {
let source = String::from("print('Hello world', 2)");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_kwargs() {
let source = String::from("my_func('positional', keyword=2)");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_elif_else() {
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = parse_program(source).unwrap();
let parse_ast = parse_program(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -144,7 +144,7 @@ mod tests {
fn test_parse_tuples() {
let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(parse_program(source).unwrap());
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
}
#[test]
@ -155,48 +155,48 @@ class Foo(A, B):
pass
def method_with_default(self, arg='default'):
pass";
insta::assert_debug_snapshot!(parse_program(source).unwrap());
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
}
#[test]
fn test_parse_dict_comprehension() {
let source = String::from("{x1: x2 for y in z}");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_list_comprehension() {
let source = String::from("[x for y in z]");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_double_list_comprehension() {
let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_generator_comprehension() {
let source = String::from("(x for y in z)");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_named_expression_generator_comprehension() {
let source = String::from("(x := y + 1 for y in z)");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_else_generator_comprehension() {
let source = String::from("(x if y else y for y in z)");
let parse_ast = parse_expression(&source).unwrap();
let parse_ast = parse_expression(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

16
parser/src/string.rs
View file

@ -88,56 +88,56 @@ mod tests {
#[test]
fn test_parse_string_concat() {
let source = String::from("'Hello ' 'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_1() {
let source = String::from("'Hello ' u'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_2() {
let source = String::from("u'Hello ' 'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_1() {
let source = String::from("'Hello ' f'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_2() {
let source = String::from("'Hello ' f'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_3() {
let source = String::from("'Hello ' f'world{\"!\"}'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_1() {
let source = String::from("u'Hello ' f'world'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_2() {
let source = String::from("u'Hello ' f'world' '!'");
let parse_ast = parse_program(&source).unwrap();
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

22
src/lib.rs
View file

@ -43,22 +43,18 @@ impl fmt::Display for CompileError {
impl CompileError {
fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
CompileError {
Self {
error: error.error.into(),
location: error.location,
source_path: error.source_path,
statement: get_statement(source, error.location),
}
}
fn from_parse(
error: rustpython_parser::error::ParseError,
source: &str,
source_path: String,
) -> Self {
CompileError {
fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
Self {
error: error.error.into(),
location: error.location,
source_path,
source_path: error.source_path,
statement: get_statement(source, error.location),
}
}
@ -83,9 +79,9 @@ pub fn compile(
compile::Mode::Eval => parser::Mode::Expression,
compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
};
let mut ast = match parser::parse(source, parser_mode) {
let mut ast = match parser::parse(source, parser_mode, &source_path) {
Ok(x) => x,
Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
Err(e) => return Err(CompileError::from_parse(e, source)),
};
if opts.optimize > 0 {
ast = ConstantOptimizer::new()
@ -101,14 +97,14 @@ pub fn compile_symtable(
mode: compile::Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
let parse_err = |e| CompileError::from_parse(e, source);
let res = match mode {
compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
let ast = parser::parse_program(source).map_err(parse_err)?;
let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
symboltable::SymbolTable::scan_program(&ast)
}
compile::Mode::Eval => {
let expr = parser::parse_expression(source).map_err(parse_err)?;
let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
symboltable::SymbolTable::scan_expr(&expr)
}
};