mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Parse Trait

Browse source
This commit is contained in:
Jeong YunWon 2023-05-28 20:45:21 +09:00
parent 5493c9f4e3
commit 9a4d586d1e
7 changed files with 278 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

5
parser/README.md
View file

@ -60,7 +60,8 @@ Execution mode check. Allowed modes are `exec`, `eval` or `single`.
For example, one could do this:
```
use rustpython_parser::{parser, ast};
use rustpython_parser::{Parse, ast};
let python_source = "print('Hello world')";
let python_ast = parser::parse_expression(python_source).unwrap();
let python_statements = ast::Suite::parse(python_source).unwrap(); // statements
let python_expr = ast::Expr::parse(python_source).unwrap(); // or expr
```

38
parser/src/context.rs
View file

@ -49,19 +49,19 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
#[cfg(test)]
mod tests {
use crate::parser::parse_program;
use crate::{ast, Parse};
#[test]
fn test_assign_name() {
let source = "x = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_tuple() {
let source = "(x, y) = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -69,35 +69,35 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_list() {
let source = "[x, y] = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_attribute() {
let source = "x.y = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_subscript() {
let source = "x[y] = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_starred() {
let source = "(x, *y) = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_for() {
let source = "for x in (1, 2, 3): pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -105,7 +105,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_list_comp() {
let source = "x = [y for y in (1, 2, 3)]";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -113,7 +113,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_set_comp() {
let source = "x = {y for y in (1, 2, 3)}";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -121,63 +121,63 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_with() {
let source = "with 1 as x: pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_named_expr() {
let source = "if x:= 1: pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_ann_assign_name() {
let source = "x: int = 1";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_name() {
let source = "x += 1";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_attribute() {
let source = "x.y += (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_subscript() {
let source = "x[y] += (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_name() {
let source = "del x";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_attribute() {
let source = "del x.y";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_subscript() {
let source = "del x[y]";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

6
parser/src/function.rs
View file

@ -155,7 +155,7 @@ const fn is_starred(exp: &ast::Expr) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::{parse_program, ParseErrorType};
use crate::{ast, parser::ParseErrorType, Parse};
#[cfg(not(feature = "all-nodes-with-ranges"))]
macro_rules! function_and_lambda {
@ -163,7 +163,7 @@ mod tests {
$(
#[test]
fn $name() {
let parse_ast = parse_program($code, "<test>");
let parse_ast = ast::Suite::parse($code, "<test>");
insta::assert_debug_snapshot!(parse_ast);
}
)*
@ -190,7 +190,7 @@ mod tests {
}
fn function_parse_error(src: &str) -> LexicalErrorType {
let parse_ast = parse_program(src, "<test>");
let parse_ast = ast::Suite::parse(src, "<test>");
parse_ast
.map_err(|e| match e.error {
ParseErrorType::Lexical(e) => e,

2
parser/src/lexer.rs
View file

@ -212,7 +212,7 @@ pub fn lex_starts_at(
source: &str,
mode: Mode,
start_offset: TextSize,
) -> impl Iterator<Item = LexResult> + '_ {
) -> SoftKeywordTransformer<Lexer<std::str::Chars<'_>>> {
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
}

12
parser/src/lib.rs
View file

@ -94,13 +94,13 @@
//! mode or tokenizing the source beforehand:
//!
//! ```
//! use rustpython_parser::parse_program;
//! use rustpython_parser::{Parse, ast};
//!
//! let python_source = r#"
//! def is_odd(i):
//! return bool(i & 1)
//! "#;
//! let ast = parse_program(python_source, "<embedded>");
//! let ast = ast::Suite::parse(python_source, "<embedded>");
//!
//! assert!(ast.is_ok());
//! ```
@ -126,13 +126,13 @@ mod soft_keywords;
mod string;
mod token;
pub use parser::{
parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
parse_tokens, ParseError, ParseErrorType,
};
pub use parser::{parse, parse_starts_at, parse_tokens, Parse, ParseError, ParseErrorType};
pub use string::FStringErrorType;
pub use token::{StringKind, Tok};
#[allow(deprecated)]
pub use parser::{parse_expression, parse_expression_starts_at, parse_program};
#[rustfmt::skip]
mod python {
#![allow(clippy::all)]

262
parser/src/parser.rs
View file

@ -13,7 +13,7 @@
//! [`Mode`]: crate::mode
use crate::{
ast,
ast::{self, OptionalRange, Ranged},
lexer::{self, LexResult, LexicalError, LexicalErrorType},
python,
text_size::TextSize,
@ -23,9 +23,179 @@ use crate::{
use itertools::Itertools;
use std::iter;
use crate::text_size::TextRange;
use crate::{lexer::Lexer, soft_keywords::SoftKeywordTransformer, text_size::TextRange};
pub(super) use lalrpop_util::ParseError as LalrpopError;
use rustpython_ast::OptionalRange;
/// Parse Python code string to implementor's type.
///
/// # Example
///
/// For example, parsing a simple function definition and a call to that function:
///
/// ```
/// use rustpython_parser::{self as parser, ast, Parse};
/// let source = r#"
/// def foo():
/// return 42
///
/// print(foo())
/// "#;
/// let program = ast::Suite::parse(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
///
/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
/// somewhat silly, location:
///
/// ```
/// use rustpython_parser::{self as parser, ast, Parse, text_size::TextSize};
///
/// let expr = ast::Expr::parse_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
pub trait Parse
where
Self: Sized,
{
fn parse(source: &str, source_path: &str) -> Result<Self, ParseError> {
Self::parse_starts_at(source, source_path, TextSize::default())
}
fn parse_starts_at(
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Self, ParseError> {
let lxr = Self::lex_starts_at(source, offset);
#[cfg(feature = "full-lexer")]
let lxr =
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
Self::parse_tokens(lxr, source_path)
}
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>>;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError>;
}
impl Parse for ast::ModModule {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Module, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Module, source_path)? {
ast::Mod::Module(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModExpression {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Expression, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Expression, source_path)? {
ast::Mod::Expression(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModInteractive {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Interactive, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Interactive, source_path)? {
ast::Mod::Interactive(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::Suite {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModModule::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Stmt {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModModule::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body;
let statement = match statements.len() {
0 => {
return Err(ParseError {
error: ParseErrorType::Eof,
offset: TextSize::default(),
source_path: source_path.to_owned(),
})
}
1 => statements.pop().unwrap(),
_ => {
return Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: statements[1].range().start(),
source_path: source_path.to_owned(),
})
}
};
Ok(statement)
}
}
impl Parse for ast::Expr {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModExpression::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body)
}
}
/// Parse a full Python program usually consisting of multiple lines.
///
@ -47,6 +217,7 @@ use rustpython_ast::OptionalRange;
/// let program = parser::parse_program(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
#[deprecated = "Use ast::Suite::parse from rustpython_parser::Parse trait."]
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
parse(source, Mode::Module, source_path).map(|top| match top {
ast::Mod::Module(ast::ModModule { body, .. }) => body,
@ -71,8 +242,9 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
/// assert!(expr.is_ok());
///
/// ```
#[deprecated = "Use ast::Expr::parse from rustpython_parser::Parse trait."]
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
parse_expression_starts_at(source, path, TextSize::default())
ast::Expr::parse(source, path)
}
/// Parses a Python expression from a given location.
@ -91,15 +263,13 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
/// ```
#[deprecated = "Use ast::Expr::parse_starts_at from rustpython_parser::Parse trait."]
pub fn parse_expression_starts_at(
source: &str,
path: &str,
offset: TextSize,
) -> Result<ast::Expr, ParseError> {
parse_starts_at(source, Mode::Expression, path, offset).map(|top| match top {
ast::Mod::Expression(ast::ModExpression { body, .. }) => *body,
_ => unreachable!(),
})
ast::Expr::parse_starts_at(source, path, offset)
}
/// Parse the given Python source code using the specified [`Mode`].
@ -188,12 +358,21 @@ pub fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
mode: Mode,
source_path: &str,
) -> Result<ast::Mod, ParseError> {
let lxr = lxr.into_iter();
#[cfg(feature = "full-lexer")]
let lxr =
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
parse_filtered_tokens(lxr, mode, source_path)
}
fn parse_filtered_tokens(
lxr: impl IntoIterator<Item = LexResult>,
mode: Mode,
source_path: &str,
) -> Result<ast::Mod, ParseError> {
let marker_token = (Tok::start_marker(mode), Default::default());
let lexer = iter::once(Ok(marker_token)).chain(lxr);
#[cfg(feature = "full-lexer")]
let lexer =
lexer.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(
lexer
@ -329,52 +508,53 @@ pub(super) fn optional_range(start: TextSize, end: TextSize) -> OptionalRange<Te
#[cfg(test)]
mod tests {
use super::*;
use crate::{ast, Parse};
#[test]
fn test_parse_empty() {
let parse_ast = parse_program("", "<test>").unwrap();
let parse_ast = ast::Suite::parse("", "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string() {
let source = "'Hello world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string() {
let source = "f'Hello world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_hello() {
let source = "print('Hello world')";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_2() {
let source = "print('Hello world', 2)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_kwargs() {
let source = "my_func('positional', keyword=2)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_elif_else() {
let source = "if 1: 10\nelif 2: 20\nelse: 30";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -382,7 +562,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -390,7 +570,7 @@ mod tests {
fn test_parse_tuples() {
let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
@ -403,14 +583,14 @@ class Foo(A, B):
def method_with_default(self, arg='default'):
pass
";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_dict_comprehension() {
let source = "{x1: x2 for y in z}";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -418,7 +598,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_list_comprehension() {
let source = "[x for y in z]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -426,7 +606,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_double_list_comprehension() {
let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -434,7 +614,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_generator_comprehension() {
let source = "(x for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -442,7 +622,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_named_expression_generator_comprehension() {
let source = "(x := y + 1 for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -450,28 +630,28 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_if_else_generator_comprehension() {
let source = "(x if y else y for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_or() {
let source = "x or y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_and() {
let source = "x and y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_slice() {
let source = "x[1:2:3]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -506,7 +686,7 @@ with (0 as a,): pass
with (0 as a, 1 as b): pass
with (0 as a, 1 as b,): pass
";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
@ -529,7 +709,7 @@ with (0 as a, 1 as b,): pass
"with a := 0 as x: pass",
"with (a := 0 as x): pass",
] {
assert!(parse_program(source, "<test>").is_err());
assert!(ast::Suite::parse(source, "<test>").is_err());
}
}
@ -541,7 +721,7 @@ array[0, *indexes, -1] = array_slice
array[*indexes_to_select, *indexes_to_select]
array[3:5, *indexes_to_select]
";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -555,13 +735,13 @@ array[3:5, *indexes_to_select]
("OFFSET %d" % offset) if offset else None,
)
)"#;
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_try() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"try:
raise ValueError(1)
except TypeError as e:
@ -576,7 +756,7 @@ except OSError as e:
#[test]
fn test_try_star() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"try:
raise ExceptionGroup("eg",
[ValueError(1), TypeError(2), OSError(3), OSError(4)])
@ -592,7 +772,7 @@ except* OSError as e:
#[test]
fn test_dict_unpacking() {
let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
let parse_ast = ast::Expr::parse(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -608,7 +788,7 @@ except* OSError as e:
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_match_as_identifier() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
match *a + b, c # ((match * a) + b), c
match *(a + b), c # (match * (a + b)), c
@ -806,14 +986,14 @@ match w := x,:
case y as v,:
z = 0
"#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_match() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
match {"test": 1}:
case {
@ -844,7 +1024,7 @@ match x:
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_variadic_generics() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
"#,

50
parser/src/string.rs
View file

@ -7,7 +7,7 @@ use crate::text_size::TextRange;
use crate::{
ast::{self, Constant, Expr},
lexer::{LexicalError, LexicalErrorType},
parser::{parse_expression_starts_at, LalrpopError, ParseError, ParseErrorType},
parser::{LalrpopError, Parse, ParseError, ParseErrorType},
token::{StringKind, Tok},
};
use itertools::Itertools;
@ -591,7 +591,7 @@ impl<'a> StringParser<'a> {
fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> {
let fstring_body = format!("({source})");
let start = location - TextSize::from(1);
parse_expression_starts_at(&fstring_body, "<fstring>", start)
ast::Expr::parse_starts_at(&fstring_body, "<fstring>", start)
}
fn parse_string(
@ -815,7 +815,7 @@ impl From<FStringError> for LalrpopError<TextSize, Tok, LexicalError> {
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::parse_program;
use crate::{ast, Parse};
fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
StringParser::new(
@ -957,63 +957,63 @@ mod tests {
#[test]
fn test_parse_string_concat() {
let source = "'Hello ' 'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_1() {
let source = "'Hello ' u'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_2() {
let source = "u'Hello ' 'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_1() {
let source = "'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_2() {
let source = "'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_3() {
let source = "'Hello ' f'world{\"!\"}'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_1() {
let source = "u'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_2() {
let source = "u'Hello ' f'world' '!'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string_triple_quotes_with_kind() {
let source = "u'''Hello, world!'''";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1021,7 +1021,7 @@ mod tests {
fn test_single_quoted_byte() {
// single quote
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1029,7 +1029,7 @@ mod tests {
fn test_double_quoted_byte() {
// double quote
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1037,35 +1037,35 @@ mod tests {
fn test_escape_char_in_byte_literal() {
// backslash does not escape
let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_1() {
let source = r"rb'\x1z'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_2() {
let source = r"rb'\\'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_escape_octet() {
let source = r##"b'\43a\4\1234'"##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_newline() {
let source = r#"f"\n{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1073,28 +1073,28 @@ mod tests {
fn test_fstring_unescaped_newline() {
let source = r#"f"""
{x}""""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_character() {
let source = r#"f"\\{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_fstring() {
let source = r#"rf"{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_triple_quoted_raw_fstring() {
let source = r#"rf"""{x}""""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1102,7 +1102,7 @@ mod tests {
fn test_fstring_line_continuation() {
let source = r#"rf"\
{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1112,7 +1112,7 @@ mod tests {
#[test]
fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = parse_program(&source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
)*