From debfca3a1101825e17e55be6647dfca1b9fc3394 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Tue, 1 Aug 2023 18:35:03 +0200 Subject: [PATCH] Remove `Parse` trait (#6235) --- crates/ruff/src/autofix/edits.rs | 12 +- crates/ruff/src/importer/insertion.rs | 6 +- crates/ruff/src/rules/eradicate/detection.rs | 6 +- .../rules/isort/rules/add_required_imports.rs | 4 +- .../mccabe/rules/function_is_too_complex.rs | 38 +- .../rules/pylint/rules/too_many_branches.rs | 5 +- .../rules/too_many_return_statements.rs | 5 +- .../rules/pylint/rules/too_many_statements.rs | 39 +- crates/ruff_benchmark/benches/parser.rs | 6 +- crates/ruff_dev/src/print_ast.rs | 5 +- crates/ruff_python_ast/tests/identifier.rs | 8 +- crates/ruff_python_ast/tests/stmt_if.rs | 18 +- crates/ruff_python_codegen/src/generator.rs | 12 +- crates/ruff_python_codegen/src/lib.rs | 5 +- .../src/expression/expr_name.rs | 6 +- .../src/statement/suite.rs | 6 +- crates/ruff_python_parser/src/context.rs | 39 +- crates/ruff_python_parser/src/function.rs | 8 +- crates/ruff_python_parser/src/lib.rs | 40 +- crates/ruff_python_parser/src/parse.rs | 1177 ----------------- crates/ruff_python_parser/src/parser.rs | 131 +- crates/ruff_python_parser/src/string.rs | 54 +- crates/ruff_python_parser/src/typing.rs | 6 +- crates/ruff_python_trivia/src/whitespace.rs | 14 +- fuzz/fuzz_targets/ruff_parse_simple.rs | 5 +- 25 files changed, 235 insertions(+), 1420 deletions(-) delete mode 100644 crates/ruff_python_parser/src/parse.rs diff --git a/crates/ruff/src/autofix/edits.rs b/crates/ruff/src/autofix/edits.rs index c2c02bdf30..5e3ff66e6c 100644 --- a/crates/ruff/src/autofix/edits.rs +++ b/crates/ruff/src/autofix/edits.rs @@ -294,8 +294,8 @@ fn next_stmt_break(semicolon: TextSize, locator: &Locator) -> TextSize { #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::{Ranged, Suite}; - use ruff_python_parser::Parse; + use ruff_python_ast::Ranged; + use ruff_python_parser::parse_suite; use ruff_text_size::TextSize; use ruff_source_file::Locator; @@ -305,13 +305,13 @@ mod tests { #[test] fn find_semicolon() -> Result<()> { let contents = "x = 1"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert_eq!(trailing_semicolon(stmt.end(), &locator), None); let contents = "x = 1; y = 1"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert_eq!( @@ -320,7 +320,7 @@ mod tests { ); let contents = "x = 1 ; y = 1"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert_eq!( @@ -333,7 +333,7 @@ x = 1 \ ; y = 1 " .trim(); - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert_eq!( diff --git a/crates/ruff/src/importer/insertion.rs b/crates/ruff/src/importer/insertion.rs index 7d5e2aaf94..5739955bed 100644 --- a/crates/ruff/src/importer/insertion.rs +++ b/crates/ruff/src/importer/insertion.rs @@ -299,12 +299,12 @@ fn match_leading_semicolon(s: &str) -> Option { #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::Suite; + use ruff_python_parser::lexer::LexResult; - use ruff_python_parser::Parse; use ruff_text_size::TextSize; use ruff_python_codegen::Stylist; + use ruff_python_parser::parse_suite; use ruff_source_file::{LineEnding, Locator}; use super::Insertion; @@ -312,7 +312,7 @@ mod tests { #[test] fn start_of_file() -> Result<()> { fn insert(contents: &str) -> Result { - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let tokens: Vec = ruff_python_parser::tokenize(contents); let locator = Locator::new(contents); let stylist = Stylist::from_tokens(&tokens, &locator); diff --git a/crates/ruff/src/rules/eradicate/detection.rs b/crates/ruff/src/rules/eradicate/detection.rs index 7cf497536d..a424d164ff 100644 --- a/crates/ruff/src/rules/eradicate/detection.rs +++ b/crates/ruff/src/rules/eradicate/detection.rs @@ -1,8 +1,8 @@ /// See: [eradicate.py](https://github.com/myint/eradicate/blob/98f199940979c94447a461d50d27862b118b282d/eradicate.py) use once_cell::sync::Lazy; use regex::Regex; -use ruff_python_ast::Suite; -use ruff_python_parser::Parse; + +use ruff_python_parser::parse_suite; static ALLOWLIST_REGEX: Lazy = Lazy::new(|| { Regex::new( @@ -79,7 +79,7 @@ pub(crate) fn comment_contains_code(line: &str, task_tags: &[String]) -> bool { } // Finally, compile the source code. - Suite::parse(&line, "").is_ok() + parse_suite(&line, "").is_ok() } /// Returns `true` if a line is probably part of some multiline code. diff --git a/crates/ruff/src/rules/isort/rules/add_required_imports.rs b/crates/ruff/src/rules/isort/rules/add_required_imports.rs index 9bcdc477e4..09ff10669d 100644 --- a/crates/ruff/src/rules/isort/rules/add_required_imports.rs +++ b/crates/ruff/src/rules/isort/rules/add_required_imports.rs @@ -1,6 +1,5 @@ use log::error; use ruff_python_ast::{self as ast, Stmt, Suite}; -use ruff_python_parser::Parse; use ruff_text_size::{TextRange, TextSize}; use ruff_diagnostics::{AlwaysAutofixableViolation, Diagnostic, Fix}; @@ -8,6 +7,7 @@ use ruff_macros::{derive_message_formats, violation}; use ruff_python_ast::helpers::is_docstring_stmt; use ruff_python_ast::imports::{Alias, AnyImport, FutureImport, Import, ImportFrom}; use ruff_python_codegen::Stylist; +use ruff_python_parser::parse_suite; use ruff_source_file::Locator; use crate::importer::Importer; @@ -138,7 +138,7 @@ pub(crate) fn add_required_imports( .required_imports .iter() .flat_map(|required_import| { - let Ok(body) = Suite::parse(required_import, "") else { + let Ok(body) = parse_suite(required_import, "") else { error!("Failed to parse required import: `{}`", required_import); return vec![]; }; diff --git a/crates/ruff/src/rules/mccabe/rules/function_is_too_complex.rs b/crates/ruff/src/rules/mccabe/rules/function_is_too_complex.rs index 4bd4041648..927ae37b5c 100644 --- a/crates/ruff/src/rules/mccabe/rules/function_is_too_complex.rs +++ b/crates/ruff/src/rules/mccabe/rules/function_is_too_complex.rs @@ -169,8 +169,8 @@ pub(crate) fn function_is_too_complex( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::Suite; - use ruff_python_parser::Parse; + + use ruff_python_parser::parse_suite; use super::get_complexity_number; @@ -180,7 +180,7 @@ mod tests { def trivial(): pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -191,7 +191,7 @@ def trivial(): def expr_as_statement(): 0xF00D "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -204,7 +204,7 @@ def sequential(n): s = k + n return s "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -220,7 +220,7 @@ def if_elif_else_dead_path(n): else: return "smaller than or equal to three" "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 3); Ok(()) } @@ -237,7 +237,7 @@ def nested_ifs(): else: return "smaller than or equal to three" "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 3); Ok(()) } @@ -249,7 +249,7 @@ def for_loop(): for i in range(10): print(i) "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 2); Ok(()) } @@ -263,7 +263,7 @@ def for_else(mylist): else: print(None) "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 2); Ok(()) } @@ -277,7 +277,7 @@ def recursive(n): else: return n "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 2); Ok(()) } @@ -294,7 +294,7 @@ def nested_functions(): a() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 3); Ok(()) } @@ -312,7 +312,7 @@ def try_else(): else: print(4) "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 4); Ok(()) } @@ -329,7 +329,7 @@ def nested_try_finally(): finally: print(3) "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -346,7 +346,7 @@ async def foobar(a, b, c): async for x in a: pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 3); Ok(()) } @@ -357,7 +357,7 @@ async def foobar(a, b, c): def annotated_assign(): x: Any = None "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -393,7 +393,7 @@ class Class: return ServiceProvider(Logger()) "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 9); Ok(()) } @@ -407,7 +407,7 @@ def process_detect_lines(): finally: pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 1); Ok(()) } @@ -422,7 +422,7 @@ def process_detect_lines(): if res: errors.append(f"Non-zero exit code {res}") "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 2); Ok(()) } @@ -435,7 +435,7 @@ def with_lock(): if foo: print('bar') "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(get_complexity_number(&stmts), 2); Ok(()) } diff --git a/crates/ruff/src/rules/pylint/rules/too_many_branches.rs b/crates/ruff/src/rules/pylint/rules/too_many_branches.rs index 466cb3e47a..bc5e459e9f 100644 --- a/crates/ruff/src/rules/pylint/rules/too_many_branches.rs +++ b/crates/ruff/src/rules/pylint/rules/too_many_branches.rs @@ -183,13 +183,12 @@ pub(crate) fn too_many_branches( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::Suite; - use ruff_python_parser::Parse; + use ruff_python_parser::parse_suite; use super::num_branches; fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> { - let branches = Suite::parse(source, "")?; + let branches = parse_suite(source, "")?; assert_eq!(num_branches(&branches), expected_num_branches); Ok(()) } diff --git a/crates/ruff/src/rules/pylint/rules/too_many_return_statements.rs b/crates/ruff/src/rules/pylint/rules/too_many_return_statements.rs index 0a1c4f40fd..2ee2abb11d 100644 --- a/crates/ruff/src/rules/pylint/rules/too_many_return_statements.rs +++ b/crates/ruff/src/rules/pylint/rules/too_many_return_statements.rs @@ -98,13 +98,12 @@ pub(crate) fn too_many_return_statements( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::Suite; - use ruff_python_parser::Parse; + use ruff_python_parser::parse_suite; use super::num_returns; fn test_helper(source: &str, expected: usize) -> Result<()> { - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_returns(&stmts), expected); Ok(()) } diff --git a/crates/ruff/src/rules/pylint/rules/too_many_statements.rs b/crates/ruff/src/rules/pylint/rules/too_many_statements.rs index 3863c251e7..0df12604ac 100644 --- a/crates/ruff/src/rules/pylint/rules/too_many_statements.rs +++ b/crates/ruff/src/rules/pylint/rules/too_many_statements.rs @@ -166,8 +166,7 @@ pub(crate) fn too_many_statements( #[cfg(test)] mod tests { use anyhow::Result; - use ruff_python_ast::Suite; - use ruff_python_parser::Parse; + use ruff_python_parser::parse_suite; use super::num_statements; @@ -177,7 +176,7 @@ mod tests { def f(): pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 2); Ok(()) } @@ -191,7 +190,7 @@ def f(): else: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 5); Ok(()) } @@ -206,7 +205,7 @@ def f(): if a: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 6); Ok(()) } @@ -220,7 +219,7 @@ def f(): elif a: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 5); Ok(()) } @@ -238,7 +237,7 @@ def f(): else: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 9); Ok(()) } @@ -267,7 +266,7 @@ async def f(): import time pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 19); Ok(()) } @@ -279,7 +278,7 @@ def f(): for i in range(10): pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 2); Ok(()) } @@ -293,7 +292,7 @@ def f(): else: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 3); Ok(()) } @@ -308,7 +307,7 @@ def f(): print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 5); Ok(()) } @@ -326,7 +325,7 @@ def f(): print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 3); Ok(()) } @@ -337,7 +336,7 @@ def f(): def f(): return "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 1); Ok(()) } @@ -353,7 +352,7 @@ def f(): print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 6); Ok(()) } @@ -367,7 +366,7 @@ def f(): except Exception: raise "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 5); Ok(()) } @@ -383,7 +382,7 @@ def f(): else: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 7); Ok(()) } @@ -401,7 +400,7 @@ def f(): finally: pass "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 10); Ok(()) } @@ -417,7 +416,7 @@ def f(): except Exception: raise "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 8); Ok(()) } @@ -435,7 +434,7 @@ def f(): finally: print() "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 11); Ok(()) } @@ -447,7 +446,7 @@ def f(): for i in range(10): yield i "#; - let stmts = Suite::parse(source, "")?; + let stmts = parse_suite(source, "")?; assert_eq!(num_statements(&stmts), 2); Ok(()) } diff --git a/crates/ruff_benchmark/benches/parser.rs b/crates/ruff_benchmark/benches/parser.rs index d7afdd106c..fa3ed71326 100644 --- a/crates/ruff_benchmark/benches/parser.rs +++ b/crates/ruff_benchmark/benches/parser.rs @@ -4,8 +4,8 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError}; use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor}; -use ruff_python_ast::{Stmt, Suite}; -use ruff_python_parser::Parse; +use ruff_python_ast::Stmt; +use ruff_python_parser::parse_suite; #[cfg(target_os = "windows")] #[global_allocator] @@ -66,7 +66,7 @@ fn benchmark_parser(criterion: &mut Criterion) { &case, |b, case| { b.iter(|| { - let parsed = Suite::parse(case.code(), case.name()).unwrap(); + let parsed = parse_suite(case.code(), case.name()).unwrap(); let mut visitor = CountVisitor { count: 0 }; visitor.visit_body(&parsed); diff --git a/crates/ruff_dev/src/print_ast.rs b/crates/ruff_dev/src/print_ast.rs index 287f969fca..fc141b13e7 100644 --- a/crates/ruff_dev/src/print_ast.rs +++ b/crates/ruff_dev/src/print_ast.rs @@ -5,8 +5,7 @@ use std::fs; use std::path::PathBuf; use anyhow::Result; -use ruff_python_ast::Suite; -use ruff_python_parser::Parse; +use ruff_python_parser::parse_suite; #[derive(clap::Args)] pub(crate) struct Args { @@ -17,7 +16,7 @@ pub(crate) struct Args { pub(crate) fn main(args: &Args) -> Result<()> { let contents = fs::read_to_string(&args.file)?; - let python_ast = Suite::parse(&contents, &args.file.to_string_lossy())?; + let python_ast = parse_suite(&contents, &args.file.to_string_lossy())?; println!("{python_ast:#?}"); Ok(()) } diff --git a/crates/ruff_python_ast/tests/identifier.rs b/crates/ruff_python_ast/tests/identifier.rs index cf24c84d00..a32dd12c26 100644 --- a/crates/ruff_python_ast/tests/identifier.rs +++ b/crates/ruff_python_ast/tests/identifier.rs @@ -1,5 +1,4 @@ -use ruff_python_ast::Stmt; -use ruff_python_parser::{Parse, ParseError}; +use ruff_python_parser::{parse_suite, ParseError}; use ruff_text_size::{TextRange, TextSize}; use ruff_python_ast::identifier; @@ -13,8 +12,9 @@ else: pass "# .trim(); - let stmt = Stmt::parse(contents, "")?; - let range = identifier::else_(&stmt, contents).unwrap(); + let stmts = parse_suite(contents, "")?; + let stmt = stmts.first().unwrap(); + let range = identifier::else_(stmt, contents).unwrap(); assert_eq!(&contents[range], "else"); assert_eq!( range, diff --git a/crates/ruff_python_ast/tests/stmt_if.rs b/crates/ruff_python_ast/tests/stmt_if.rs index 178ec28dbc..73d156ae09 100644 --- a/crates/ruff_python_ast/tests/stmt_if.rs +++ b/crates/ruff_python_ast/tests/stmt_if.rs @@ -1,6 +1,6 @@ use ruff_python_ast::stmt_if::elif_else_range; -use ruff_python_ast::Stmt; -use ruff_python_parser::{Parse, ParseError}; + +use ruff_python_parser::{parse_suite, ParseError}; use ruff_text_size::TextSize; #[test] @@ -10,8 +10,11 @@ fn extract_elif_else_range() -> Result<(), ParseError> { elif b: ... "; - let stmt = Stmt::parse(contents, "")?; - let stmt = Stmt::as_if_stmt(&stmt).unwrap(); + let mut stmts = parse_suite(contents, "")?; + let stmt = stmts + .pop() + .and_then(ruff_python_ast::Stmt::if_stmt) + .unwrap(); let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.end(), TextSize::from(18)); @@ -21,8 +24,11 @@ elif b: else: ... "; - let stmt = Stmt::parse(contents, "")?; - let stmt = Stmt::as_if_stmt(&stmt).unwrap(); + let mut stmts = parse_suite(contents, "")?; + let stmt = stmts + .pop() + .and_then(ruff_python_ast::Stmt::if_stmt) + .unwrap(); let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.end(), TextSize::from(18)); diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index a48416c21a..bec25bcf20 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -1503,8 +1503,8 @@ impl<'a> Generator<'a> { #[cfg(test)] mod tests { - use ruff_python_ast::{Mod, ModModule, Stmt}; - use ruff_python_parser::{self, Mode, Parse}; + use ruff_python_ast::{Mod, ModModule}; + use ruff_python_parser::{self, parse_suite, Mode}; use ruff_source_file::LineEnding; @@ -1515,9 +1515,9 @@ mod tests { let indentation = Indentation::default(); let quote = Quote::default(); let line_ending = LineEnding::default(); - let stmt = Stmt::parse(contents, "").unwrap(); + let stmt = parse_suite(contents, "").unwrap(); let mut generator = Generator::new(&indentation, quote, line_ending); - generator.unparse_stmt(&stmt); + generator.unparse_suite(&stmt); generator.generate() } @@ -1527,9 +1527,9 @@ mod tests { line_ending: LineEnding, contents: &str, ) -> String { - let stmt = Stmt::parse(contents, "").unwrap(); + let stmt = parse_suite(contents, "").unwrap(); let mut generator = Generator::new(indentation, quote, line_ending); - generator.unparse_stmt(&stmt); + generator.unparse_suite(&stmt); generator.generate() } diff --git a/crates/ruff_python_codegen/src/lib.rs b/crates/ruff_python_codegen/src/lib.rs index 067d1d5b3e..a0bf9fcd4f 100644 --- a/crates/ruff_python_codegen/src/lib.rs +++ b/crates/ruff_python_codegen/src/lib.rs @@ -2,15 +2,14 @@ mod generator; mod stylist; pub use generator::Generator; -use ruff_python_ast::Suite; -use ruff_python_parser::{lexer, Mode, Parse, ParseError}; +use ruff_python_parser::{lexer, parse_suite, Mode, ParseError}; use ruff_source_file::Locator; pub use stylist::{Quote, Stylist}; /// Run round-trip source code generation on a given Python code. pub fn round_trip(code: &str, source_path: &str) -> Result { let locator = Locator::new(code); - let python_ast = Suite::parse(code, source_path)?; + let python_ast = parse_suite(code, source_path)?; let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect(); let stylist = Stylist::from_tokens(&tokens, &locator); let mut generator: Generator = (&stylist).into(); diff --git a/crates/ruff_python_formatter/src/expression/expr_name.rs b/crates/ruff_python_formatter/src/expression/expr_name.rs index e9aba4f888..2653f343b4 100644 --- a/crates/ruff_python_formatter/src/expression/expr_name.rs +++ b/crates/ruff_python_formatter/src/expression/expr_name.rs @@ -36,13 +36,13 @@ impl NeedsParentheses for ExprName { #[cfg(test)] mod tests { - use ruff_python_ast::{ModModule, Ranged}; - use ruff_python_parser::Parse; + use ruff_python_ast::Ranged; + use ruff_python_parser::parse_program; use ruff_text_size::{TextRange, TextSize}; #[test] fn name_range_with_comments() { - let source = ModModule::parse("a # comment", "file.py").unwrap(); + let source = parse_program("a # comment", "file.py").unwrap(); let expression_statement = source .body diff --git a/crates/ruff_python_formatter/src/statement/suite.rs b/crates/ruff_python_formatter/src/statement/suite.rs index 0b68b8df29..b021d8c690 100644 --- a/crates/ruff_python_formatter/src/statement/suite.rs +++ b/crates/ruff_python_formatter/src/statement/suite.rs @@ -210,8 +210,8 @@ impl<'ast> IntoFormat> for Suite { #[cfg(test)] mod tests { use ruff_formatter::format; - use ruff_python_ast::Suite; - use ruff_python_parser::Parse; + + use ruff_python_parser::parse_suite; use crate::comments::Comments; use crate::prelude::*; @@ -240,7 +240,7 @@ def trailing_func(): pass "#; - let statements = Suite::parse(source, "test.py").unwrap(); + let statements = parse_suite(source, "test.py").unwrap(); let context = PyFormatContext::new(PyFormatOptions::default(), source, Comments::default()); diff --git a/crates/ruff_python_parser/src/context.rs b/crates/ruff_python_parser/src/context.rs index 683c35a30a..83aab32a1f 100644 --- a/crates/ruff_python_parser/src/context.rs +++ b/crates/ruff_python_parser/src/context.rs @@ -49,132 +49,131 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr { #[cfg(test)] mod tests { - use crate::Parse; - use ruff_python_ast as ast; + use crate::parser::parse_suite; #[test] fn test_assign_name() { let source = "x = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_tuple() { let source = "(x, y) = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_list() { let source = "[x, y] = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_attribute() { let source = "x.y = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_subscript() { let source = "x[y] = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_starred() { let source = "(x, *y) = (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_for() { let source = "for x in (1, 2, 3): pass"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_list_comp() { let source = "x = [y for y in (1, 2, 3)]"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_set_comp() { let source = "x = {y for y in (1, 2, 3)}"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_with() { let source = "with 1 as x: pass"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_named_expr() { let source = "if x:= 1: pass"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_ann_assign_name() { let source = "x: int = 1"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_name() { let source = "x += 1"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_attribute() { let source = "x.y += (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_subscript() { let source = "x[y] += (1, 2, 3)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_name() { let source = "del x"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_attribute() { let source = "del x.y"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_subscript() { let source = "del x[y]"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } } diff --git a/crates/ruff_python_parser/src/function.rs b/crates/ruff_python_parser/src/function.rs index a26ef6395e..c325855f52 100644 --- a/crates/ruff_python_parser/src/function.rs +++ b/crates/ruff_python_parser/src/function.rs @@ -133,15 +133,15 @@ const fn is_starred(exp: &ast::Expr) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::{Parse, ParseErrorType}; - use ruff_python_ast::{self as ast}; + use crate::parser::parse_suite; + use crate::ParseErrorType; macro_rules! function_and_lambda { ($($name:ident: $code:expr,)*) => { $( #[test] fn $name() { - let parse_ast = ast::Suite::parse($code, ""); + let parse_ast = crate::parser::parse_suite($code, ""); insta::assert_debug_snapshot!(parse_ast); } )* @@ -172,7 +172,7 @@ mod tests { } fn function_parse_error(src: &str) -> LexicalErrorType { - let parse_ast = ast::Suite::parse(src, ""); + let parse_ast = parse_suite(src, ""); parse_ast .map_err(|e| match e.error { ParseErrorType::Lexical(e) => e, diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index ac00c8ddcb..0697099a43 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -94,14 +94,13 @@ //! mode or tokenizing the source beforehand: //! //! ``` -//! use ruff_python_parser::{Parse}; -//! use ruff_python_ast as ast; +//! use ruff_python_parser::parse_suite; //! //! let python_source = r#" //! def is_odd(i): //! return bool(i & 1) //! "#; -//! let ast = ast::Suite::parse(python_source, ""); +//! let ast = parse_suite(python_source, ""); //! //! assert!(ast.is_ok()); //! ``` @@ -111,11 +110,11 @@ //! [lexer]: crate::lexer use crate::lexer::LexResult; -pub use parse::Parse; -pub use parser::{parse, parse_starts_at, parse_tokens, ParseError, ParseErrorType}; -#[allow(deprecated)] -pub use parser::{parse_expression, parse_expression_starts_at, parse_program}; -use ruff_python_ast::{CmpOp, Expr, ModModule, Ranged, Suite}; +pub use parser::{ + parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at, + parse_suite, parse_tokens, ParseError, ParseErrorType, +}; +use ruff_python_ast::{CmpOp, Expr, Mod, Ranged, Suite}; use ruff_text_size::{TextRange, TextSize}; pub use string::FStringErrorType; pub use token::{StringKind, Tok, TokenKind}; @@ -124,7 +123,6 @@ mod function; // Skip flattening lexer to distinguish from full ruff_python_parser mod context; pub mod lexer; -mod parse; mod parser; mod soft_keywords; mod string; @@ -149,7 +147,10 @@ pub fn parse_program_tokens( lxr: Vec, source_path: &str, ) -> anyhow::Result { - ModModule::parse_tokens(lxr, source_path).map(|module| module.body) + match parse_tokens(lxr, Mode::Module, source_path)? { + Mod::Module(m) => Ok(m.body), + Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"), + } } /// Return the `Range` of the first `Tok::Colon` token in a `Range`. @@ -344,11 +345,10 @@ mod python { #[cfg(test)] mod tests { - use crate::Parse; - use crate::{first_colon_range, locate_cmp_ops, LocatedCmpOp}; + use crate::{first_colon_range, locate_cmp_ops, parse_expression, LocatedCmpOp}; use anyhow::Result; use ruff_python_ast::CmpOp; - use ruff_python_ast::Expr; + use ruff_text_size::{TextLen, TextRange, TextSize}; #[test] @@ -366,7 +366,7 @@ mod tests { #[test] fn extract_cmp_op_location() -> Result<()> { let contents = "x == 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -376,7 +376,7 @@ mod tests { ); let contents = "x != 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -386,7 +386,7 @@ mod tests { ); let contents = "x is 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -396,7 +396,7 @@ mod tests { ); let contents = "x is not 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -406,7 +406,7 @@ mod tests { ); let contents = "x in 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -416,7 +416,7 @@ mod tests { ); let contents = "x not in 1"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( @@ -426,7 +426,7 @@ mod tests { ); let contents = "x != (1 is not 2)"; - let expr = Expr::parse(contents, "")?; + let expr = parse_expression(contents, "")?; assert_eq!( locate_cmp_ops(&expr, contents), vec![LocatedCmpOp::new( diff --git a/crates/ruff_python_parser/src/parse.rs b/crates/ruff_python_parser/src/parse.rs deleted file mode 100644 index ab6ab7d5a1..0000000000 --- a/crates/ruff_python_parser/src/parse.rs +++ /dev/null @@ -1,1177 +0,0 @@ -use crate::lexer::{lex, lex_starts_at, LexResult}; -use crate::{parse_tokens, Mode, ParseError, ParseErrorType}; -use ruff_python_ast as ast; -use ruff_python_ast::Ranged; -use ruff_text_size::TextSize; - -/// Parse Python code string to implementor's type. -/// -/// # Example -/// -/// For example, parsing a simple function definition and a call to that function: -/// -/// ``` -/// use ruff_python_parser::{self as parser, Parse}; -/// use ruff_python_ast as ast; -/// let source = r#" -/// def foo(): -/// return 42 -/// -/// print(foo()) -/// "#; -/// let program = ast::Suite::parse(source, ""); -/// assert!(program.is_ok()); -/// ``` -/// -/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different, -/// somewhat silly, location: -/// -/// ``` -/// # use ruff_text_size::TextSize; -/// # use ruff_python_ast as ast; -/// # use ruff_python_parser::{self as parser, Parse}; -/// -/// let expr = ast::Expr::parse_starts_at("1 + 2", "", TextSize::from(400)); -/// assert!(expr.is_ok()); -pub trait Parse -where - Self: Sized, -{ - const MODE: Mode; - - fn parse(source: &str, source_path: &str) -> Result { - let tokens = lex(source, Self::MODE); - - Self::parse_tokens(tokens, source_path) - } - - fn parse_without_path(source: &str) -> Result { - Self::parse(source, "") - } - - fn parse_starts_at( - source: &str, - source_path: &str, - offset: TextSize, - ) -> Result { - let tokens = lex_starts_at(source, Self::MODE, offset); - - Self::parse_tokens(tokens, source_path) - } - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result; -} - -impl Parse for ast::ModModule { - const MODE: Mode = Mode::Module; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - match parse_tokens(lxr, Mode::Module, source_path)? { - ast::Mod::Module(m) => Ok(m), - ast::Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"), - } - } -} - -impl Parse for ast::ModExpression { - const MODE: Mode = Mode::Expression; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - match parse_tokens(lxr, Mode::Expression, source_path)? { - ast::Mod::Expression(m) => Ok(m), - ast::Mod::Module(_) => unreachable!("Mode::Module doesn't return other variant"), - } - } -} - -impl Parse for ast::Suite { - const MODE: Mode = Mode::Module; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body) - } -} - -impl Parse for ast::Stmt { - const MODE: Mode = Mode::Module; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body; - let statement = match statements.len() { - 0 => { - return Err(ParseError { - error: ParseErrorType::Eof, - offset: TextSize::default(), - source_path: source_path.to_owned(), - }) - } - 1 => statements.pop().unwrap(), - _ => { - return Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: statements[1].range().start(), - source_path: source_path.to_owned(), - }) - } - }; - Ok(statement) - } -} - -impl Parse for ast::Expr { - const MODE: Mode = Mode::Expression; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body) - } -} - -impl Parse for ast::Identifier { - const MODE: Mode = Mode::Expression; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let expr = ast::Expr::parse_tokens(lxr, source_path)?; - match expr { - ast::Expr::Name(name) => { - let range = name.range(); - Ok(ast::Identifier::new(name.id, range)) - } - expr => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: expr.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::Constant { - const MODE: Mode = Mode::Expression; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let expr = ast::Expr::parse_tokens(lxr, source_path)?; - match expr { - ast::Expr::Constant(c) => Ok(c.value), - expr => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: expr.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtFunctionDef { - const MODE: Mode = Mode::Module; - - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::FunctionDef(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAsyncFunctionDef { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::AsyncFunctionDef(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtClassDef { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::ClassDef(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtReturn { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Return(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtDelete { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Delete(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAssign { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Assign(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtTypeAlias { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::TypeAlias(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAugAssign { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::AugAssign(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAnnAssign { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::AnnAssign(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtFor { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::For(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAsyncFor { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::AsyncFor(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtWhile { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::While(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtIf { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::If(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtWith { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::With(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAsyncWith { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::AsyncWith(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtMatch { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Match(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtRaise { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Raise(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtTry { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Try(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtTryStar { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::TryStar(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtAssert { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Assert(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtImport { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Import(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtImportFrom { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::ImportFrom(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtGlobal { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Global(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtNonlocal { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Nonlocal(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtExpr { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Expr(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtPass { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Pass(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtBreak { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Break(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::StmtContinue { - const MODE: Mode = Mode::Module; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Stmt::parse_tokens(lxr, source_path)?; - match node { - ast::Stmt::Continue(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprBoolOp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::BoolOp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprNamedExpr { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::NamedExpr(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprBinOp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::BinOp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprUnaryOp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::UnaryOp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprLambda { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Lambda(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprIfExp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::IfExp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprDict { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Dict(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprSet { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Set(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprListComp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::ListComp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprSetComp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::SetComp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprDictComp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::DictComp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprGeneratorExp { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::GeneratorExp(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprAwait { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Await(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprYield { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Yield(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprYieldFrom { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::YieldFrom(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprCompare { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Compare(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprCall { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Call(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprFormattedValue { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::FormattedValue(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprJoinedStr { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::JoinedStr(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprConstant { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Constant(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprAttribute { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Attribute(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprSubscript { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Subscript(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprStarred { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Starred(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprName { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Name(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprList { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::List(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprTuple { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Tuple(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} - -impl Parse for ast::ExprSlice { - const MODE: Mode = Mode::Expression; - fn parse_tokens( - lxr: impl IntoIterator, - source_path: &str, - ) -> Result { - let node = ast::Expr::parse_tokens(lxr, source_path)?; - match node { - ast::Expr::Slice(node) => Ok(node), - node => Err(ParseError { - error: ParseErrorType::InvalidToken, - offset: node.range().start(), - source_path: source_path.to_owned(), - }), - } - } -} diff --git a/crates/ruff_python_parser/src/parser.rs b/crates/ruff_python_parser/src/parser.rs index df09c8e5e3..d9edbbd7b1 100644 --- a/crates/ruff_python_parser/src/parser.rs +++ b/crates/ruff_python_parser/src/parser.rs @@ -18,14 +18,15 @@ use itertools::Itertools; pub(super) use lalrpop_util::ParseError as LalrpopError; use ruff_text_size::{TextRange, TextSize}; +use crate::lexer::{lex, lex_starts_at}; use crate::{ lexer::{self, LexResult, LexicalError, LexicalErrorType}, python, token::Tok, - Mode, Parse, + Mode, }; use ruff_python_ast as ast; -use ruff_python_ast::ModModule; +use ruff_python_ast::{Mod, ModModule, Suite}; /// Parse a full Python program usually consisting of multiple lines. /// @@ -47,9 +48,16 @@ use ruff_python_ast::ModModule; /// let program = parser::parse_program(source, ""); /// assert!(program.is_ok()); /// ``` -#[deprecated = "Use ruff_python_ast::Suite::parse from ruff_python_parser::Parse trait."] -pub fn parse_program(source: &str, source_path: &str) -> Result { - ModModule::parse(source, source_path).map(|module| module.body) +pub fn parse_program(source: &str, source_path: &str) -> Result { + let lexer = lex(source, Mode::Module); + match parse_tokens(lexer, Mode::Module, source_path)? { + Mod::Module(m) => Ok(m), + Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"), + } +} + +pub fn parse_suite(source: &str, source_path: &str) -> Result { + parse_program(source, source_path).map(|m| m.body) } /// Parses a single Python expression. @@ -68,9 +76,12 @@ pub fn parse_program(source: &str, source_path: &str) -> Result Result { - ast::Expr::parse(source, path) +pub fn parse_expression(source: &str, source_path: &str) -> Result { + let lexer = lex(source, Mode::Expression); + match parse_tokens(lexer, Mode::Expression, source_path)? { + Mod::Expression(expression) => Ok(*expression.body), + Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"), + } } /// Parses a Python expression from a given location. @@ -90,13 +101,16 @@ pub fn parse_expression(source: &str, path: &str) -> Result", TextSize::from(400)); /// assert!(expr.is_ok()); /// ``` -#[deprecated = "Use ruff_python_ast::Expr::parse_starts_at from ruff_python_parser::Parse trait."] pub fn parse_expression_starts_at( source: &str, - path: &str, + source_path: &str, offset: TextSize, ) -> Result { - ast::Expr::parse_starts_at(source, path, offset) + let lexer = lex_starts_at(source, Mode::Module, offset); + match parse_tokens(lexer, Mode::Expression, source_path)? { + Mod::Expression(expression) => Ok(*expression.body), + Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"), + } } /// Parse the given Python source code using the specified [`Mode`]. @@ -384,71 +398,69 @@ impl ParseErrorType { #[cfg(test)] mod tests { - use crate::Parse; use insta::assert_debug_snapshot; - use ruff_python_ast as ast; use super::*; #[test] fn test_parse_empty() { - let parse_ast = ast::Suite::parse("", "").unwrap(); + let parse_ast = parse_suite("", "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_string() { let source = "'Hello world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string() { let source = "f'Hello world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_hello() { let source = "print('Hello world')"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_2() { let source = "print('Hello world', 2)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_kwargs() { let source = "my_func('positional', keyword=2)"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_elif_else() { let source = "if 1: 10\nelif 2: 20\nelse: 30"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_lambda() { let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_lambda_no_args() { let source = "lambda: 1"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -456,7 +468,7 @@ mod tests { fn test_parse_tuples() { let source = "a, b = 4, 5"; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -468,7 +480,7 @@ class Foo(A, B): def method_with_default(self, arg='default'): pass "; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -499,7 +511,7 @@ class Foo[**P](): ... class Foo[X, Y: str, *U, **P](): pass "; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] fn test_parse_function_definition() { @@ -525,69 +537,69 @@ def func[**P](*args: P.args, **kwargs: P.kwargs): def func[T, U: str, *Ts, **P](): pass "; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] fn test_parse_dict_comprehension() { let source = "{x1: x2 for y in z}"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_list_comprehension() { let source = "[x for y in z]"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_double_list_comprehension() { let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_generator_comprehension() { let source = "(x for y in z)"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_named_expression_generator_comprehension() { let source = "(x := y + 1 for y in z)"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_else_generator_comprehension() { let source = "(x if y else y for y in z)"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_bool_op_or() { let source = "x or y"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_bool_op_and() { let source = "x and y"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_slice() { let source = "x[1:2:3]"; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -621,7 +633,7 @@ with (0 as a,): pass with (0 as a, 1 as b): pass with (0 as a, 1 as b,): pass "; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -644,7 +656,7 @@ with (0 as a, 1 as b,): pass "with a := 0 as x: pass", "with (a := 0 as x): pass", ] { - assert!(ast::Suite::parse(source, "").is_err()); + assert!(parse_suite(source, "").is_err()); } } @@ -656,7 +668,7 @@ array[0, *indexes, -1] = array_slice array[*indexes_to_select, *indexes_to_select] array[3:5, *indexes_to_select] "; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -669,13 +681,13 @@ array[3:5, *indexes_to_select] ("OFFSET %d" % offset) if offset else None, ) )"#; - let parse_ast = ast::Expr::parse(source, "").unwrap(); + let parse_ast = parse_expression(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_try() { - let parse_ast = ast::Suite::parse( + let parse_ast = parse_suite( r#"try: raise ValueError(1) except TypeError as e: @@ -690,7 +702,7 @@ except OSError as e: #[test] fn test_try_star() { - let parse_ast = ast::Suite::parse( + let parse_ast = parse_suite( r#"try: raise ExceptionGroup("eg", [ValueError(1), TypeError(2), OSError(3), OSError(4)]) @@ -706,7 +718,7 @@ except* OSError as e: #[test] fn test_dict_unpacking() { - let parse_ast = ast::Expr::parse(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); + let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -758,7 +770,7 @@ type X \ type X[T] \ = T "#; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -795,7 +807,7 @@ type = 1 type = x = 1 x = type = 1 "#; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -820,7 +832,7 @@ x = 10000 x = 133333 "#; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -846,7 +858,7 @@ if 10 .real: y = 100[no] y = 100(no) "#; - assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -874,7 +886,7 @@ match match: match = lambda query: query == event print(match(12)) "#; - insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); + insta::assert_debug_snapshot!(parse_suite(source, "").unwrap()); } #[test] @@ -1044,13 +1056,13 @@ match w := x,: case y as v,: z = 0 "#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_match() { - let parse_ast = ast::Suite::parse( + let parse_ast = parse_suite( r#" match {"test": 1}: case { @@ -1080,7 +1092,7 @@ match x: #[test] fn test_variadic_generics() { - let parse_ast = ast::Suite::parse( + let parse_ast = parse_suite( r#" def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... "#, @@ -1090,26 +1102,9 @@ def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... insta::assert_debug_snapshot!(parse_ast); } - #[test] - fn test_parse_constant() { - use num_traits::ToPrimitive; - - let c = ast::Constant::parse_without_path("'string'").unwrap(); - assert_eq!(c.str().unwrap(), "string"); - - let c = ast::Constant::parse_without_path("10").unwrap(); - assert_eq!(c.int().unwrap().to_i32().unwrap(), 10); - } - - #[test] - fn test_parse_identifier() { - let i = ast::Identifier::parse_without_path("test").unwrap(); - assert_eq!(i.as_str(), "test"); - } - #[test] fn decorator_ranges() { - let parse_ast = ast::Suite::parse( + let parse_ast = parse_suite( r#" @my_decorator def test(): diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index 5d212c3a45..b6aea76f5d 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -11,9 +11,9 @@ use ruff_text_size::{TextLen, TextRange, TextSize}; // we have to do the parsing here, manually. use crate::{ lexer::{LexicalError, LexicalErrorType}, + parse_expression_starts_at, parser::{ParseError, ParseErrorType}, token::{StringKind, Tok}, - Parse, }; // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 @@ -550,7 +550,7 @@ impl<'a> StringParser<'a> { fn parse_fstring_expr(source: &str, location: TextSize) -> Result { let fstring_body = format!("({source})"); - ast::Expr::parse_starts_at(&fstring_body, "", location) + parse_expression_starts_at(&fstring_body, "", location) } fn parse_string( @@ -785,10 +785,8 @@ impl From for crate::parser::LalrpopError Result, LexicalError> { StringParser::new(source, StringKind::FString, false, TextSize::default()).parse() @@ -926,63 +924,63 @@ mod tests { #[test] fn test_parse_string_concat() { let source = "'Hello ' 'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_1() { let source = "'Hello ' u'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_2() { let source = "u'Hello ' 'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_1() { let source = "'Hello ' f'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_2() { let source = "'Hello ' f'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_3() { let source = "'Hello ' f'world{\"!\"}'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_1() { let source = "u'Hello ' f'world'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_2() { let source = "u'Hello ' f'world' '!'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_string_triple_quotes_with_kind() { let source = "u'''Hello, world!'''"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -990,7 +988,7 @@ mod tests { fn test_single_quoted_byte() { // single quote let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -998,7 +996,7 @@ mod tests { fn test_double_quoted_byte() { // double quote let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1006,42 +1004,42 @@ mod tests { fn test_escape_char_in_byte_literal() { // backslash does not escape let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_byte_literal_1() { let source = r"rb'\x1z'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_byte_literal_2() { let source = r"rb'\\'"; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_escape_octet() { let source = r##"b'\43a\4\1234'"##; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_escaped_newline() { let source = r#"f"\n{x}""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_constant_range() { let source = r#"f"aaa{bbb}ccc{ddd}eee""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1049,28 +1047,28 @@ mod tests { fn test_fstring_unescaped_newline() { let source = r#"f""" {x}""""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_escaped_character() { let source = r#"f"\\{x}""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_fstring() { let source = r#"rf"{x}""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_triple_quoted_raw_fstring() { let source = r#"rf"""{x}""""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1078,7 +1076,7 @@ mod tests { fn test_fstring_line_continuation() { let source = r#"rf"\ {x}""#; - let parse_ast = ast::Suite::parse(source, "").unwrap(); + let parse_ast = parse_suite(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1088,7 +1086,7 @@ mod tests { #[test] fn $name() { let source = format!(r#""\N{{{0}}}""#, $alias); - let parse_ast = ast::Suite::parse(&source, "").unwrap(); + let parse_ast = parse_suite(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } )* diff --git a/crates/ruff_python_parser/src/typing.rs b/crates/ruff_python_parser/src/typing.rs index 075da24419..f6810bc54c 100644 --- a/crates/ruff_python_parser/src/typing.rs +++ b/crates/ruff_python_parser/src/typing.rs @@ -1,4 +1,4 @@ -use crate::Parse; +use crate::{parse_expression, parse_expression_starts_at}; use anyhow::Result; use ruff_python_ast::relocate::relocate_expr; use ruff_python_ast::str; @@ -33,7 +33,7 @@ pub fn parse_type_annotation( // isn't the case, e.g., for implicit concatenations, or for annotations that contain // escaped quotes. let leading_quote = str::leading_quote(expression).unwrap(); - let expr = Expr::parse_starts_at( + let expr = parse_expression_starts_at( value, "", range.start() + leading_quote.text_len(), @@ -41,7 +41,7 @@ pub fn parse_type_annotation( Ok((expr, AnnotationKind::Simple)) } else { // Otherwise, consider this a "complex" annotation. - let mut expr = Expr::parse(value, "")?; + let mut expr = parse_expression(value, "")?; relocate_expr(&mut expr, range); Ok((expr, AnnotationKind::Complex)) } diff --git a/crates/ruff_python_trivia/src/whitespace.rs b/crates/ruff_python_trivia/src/whitespace.rs index cc8f5563b7..7e23c4f3e7 100644 --- a/crates/ruff_python_trivia/src/whitespace.rs +++ b/crates/ruff_python_trivia/src/whitespace.rs @@ -83,32 +83,32 @@ impl PythonWhitespace for str { #[cfg(test)] mod tests { use crate::has_trailing_content; - use ruff_python_ast::{Ranged, Suite}; - use ruff_python_parser::{Parse, ParseError}; + use ruff_python_ast::Ranged; + use ruff_python_parser::{parse_suite, ParseError}; use ruff_source_file::Locator; #[test] fn trailing_content() -> Result<(), ParseError> { let contents = "x = 1"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); let contents = "x = 1; y = 2"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert!(has_trailing_content(stmt.end(), &locator)); let contents = "x = 1 "; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); let contents = "x = 1 # Comment"; - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); @@ -118,7 +118,7 @@ x = 1 y = 2 "# .trim(); - let program = Suite::parse(contents, "")?; + let program = parse_suite(contents, "")?; let stmt = program.first().unwrap(); let locator = Locator::new(contents); assert!(!has_trailing_content(stmt.end(), &locator)); diff --git a/fuzz/fuzz_targets/ruff_parse_simple.rs b/fuzz/fuzz_targets/ruff_parse_simple.rs index 3cbdc62491..4c23c26528 100644 --- a/fuzz/fuzz_targets/ruff_parse_simple.rs +++ b/fuzz/fuzz_targets/ruff_parse_simple.rs @@ -4,9 +4,8 @@ #![no_main] use libfuzzer_sys::{fuzz_target, Corpus}; -use ruff_python_ast::Suite; use ruff_python_codegen::{Generator, Stylist}; -use ruff_python_parser::{lexer, Mode, Parse, ParseError}; +use ruff_python_parser::{lexer, parse_suite, Mode, ParseError}; use ruff_source_file::Locator; fn do_fuzz(case: &[u8]) -> Corpus { @@ -16,7 +15,7 @@ fn do_fuzz(case: &[u8]) -> Corpus { // just round-trip it once to trigger both parse and unparse let locator = Locator::new(code); - let python_ast = match Suite::parse(code, "fuzzed-source.py") { + let python_ast = match parse_suite(code, "fuzzed-source.py") { Ok(stmts) => stmts, Err(ParseError { offset, .. }) => { let offset = offset.to_usize();