language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-24 19:12:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Remove Parse trait (#6235)

Browse source
This commit is contained in:
Micha Reiser 2023-08-01 18:35:03 +02:00 • committed by GitHub
parent 83fe103d6e
commit debfca3a11
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 235 additions and 1420 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/ruff/src/autofix/edits.rs
View file

@ -294,8 +294,8 @@ fn next_stmt_break(semicolon: TextSize, locator: &Locator) -> TextSize {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::{Ranged, Suite}; use ruff_python_ast::Ranged;
use ruff_python_parser::Parse; use ruff_python_parser::parse_suite;
use ruff_text_size::TextSize; use ruff_text_size::TextSize;
use ruff_source_file::Locator; use ruff_source_file::Locator;
@ -305,13 +305,13 @@ mod tests {
#[test] #[test]
fn find_semicolon() -> Result<()> { fn find_semicolon() -> Result<()> {
let contents = "x = 1"; let contents = "x = 1";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!(trailing_semicolon(stmt.end(), &locator), None); assert_eq!(trailing_semicolon(stmt.end(), &locator), None);
let contents = "x = 1; y = 1"; let contents = "x = 1; y = 1";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(
@ -320,7 +320,7 @@ mod tests {
); );
let contents = "x = 1 ; y = 1"; let contents = "x = 1 ; y = 1";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(
@ -333,7 +333,7 @@ x = 1 \
; y = 1 ; y = 1
" "
.trim(); .trim();
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(

6
crates/ruff/src/importer/insertion.rs
View file

@ -299,12 +299,12 @@ fn match_leading_semicolon(s: &str) -> Option<TextSize> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite;
use ruff_python_parser::lexer::LexResult; use ruff_python_parser::lexer::LexResult;
use ruff_python_parser::Parse;
use ruff_text_size::TextSize; use ruff_text_size::TextSize;
use ruff_python_codegen::Stylist; use ruff_python_codegen::Stylist;
use ruff_python_parser::parse_suite;
use ruff_source_file::{LineEnding, Locator}; use ruff_source_file::{LineEnding, Locator};
use super::Insertion; use super::Insertion;
@ -312,7 +312,7 @@ mod tests {
#[test] #[test]
fn start_of_file() -> Result<()> { fn start_of_file() -> Result<()> {
fn insert(contents: &str) -> Result<Insertion> { fn insert(contents: &str) -> Result<Insertion> {
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let tokens: Vec<LexResult> = ruff_python_parser::tokenize(contents); let tokens: Vec<LexResult> = ruff_python_parser::tokenize(contents);
let locator = Locator::new(contents); let locator = Locator::new(contents);
let stylist = Stylist::from_tokens(&tokens, &locator); let stylist = Stylist::from_tokens(&tokens, &locator);

6
crates/ruff/src/rules/eradicate/detection.rs
View file

@ -1,8 +1,8 @@
/// See: [eradicate.py](https://github.com/myint/eradicate/blob/98f199940979c94447a461d50d27862b118b282d/eradicate.py) /// See: [eradicate.py](https://github.com/myint/eradicate/blob/98f199940979c94447a461d50d27862b118b282d/eradicate.py)
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use ruff_python_ast::Suite;
use ruff_python_parser::Parse; use ruff_python_parser::parse_suite;
static ALLOWLIST_REGEX: Lazy<Regex> = Lazy::new(|| { static ALLOWLIST_REGEX: Lazy<Regex> = Lazy::new(|| {
Regex::new( Regex::new(
@ -79,7 +79,7 @@ pub(crate) fn comment_contains_code(line: &str, task_tags: &[String]) -> bool {
} }
// Finally, compile the source code. // Finally, compile the source code.
Suite::parse(&line, "<filename>").is_ok() parse_suite(&line, "<filename>").is_ok()
} }
/// Returns `true` if a line is probably part of some multiline code. /// Returns `true` if a line is probably part of some multiline code.

4
crates/ruff/src/rules/isort/rules/add_required_imports.rs
View file

@ -1,6 +1,5 @@
use log::error; use log::error;
use ruff_python_ast::{self as ast, Stmt, Suite}; use ruff_python_ast::{self as ast, Stmt, Suite};
use ruff_python_parser::Parse;
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
use ruff_diagnostics::{AlwaysAutofixableViolation, Diagnostic, Fix}; use ruff_diagnostics::{AlwaysAutofixableViolation, Diagnostic, Fix};
@ -8,6 +7,7 @@ use ruff_macros::{derive_message_formats, violation};
use ruff_python_ast::helpers::is_docstring_stmt; use ruff_python_ast::helpers::is_docstring_stmt;
use ruff_python_ast::imports::{Alias, AnyImport, FutureImport, Import, ImportFrom}; use ruff_python_ast::imports::{Alias, AnyImport, FutureImport, Import, ImportFrom};
use ruff_python_codegen::Stylist; use ruff_python_codegen::Stylist;
use ruff_python_parser::parse_suite;
use ruff_source_file::Locator; use ruff_source_file::Locator;
use crate::importer::Importer; use crate::importer::Importer;
@ -138,7 +138,7 @@ pub(crate) fn add_required_imports(
.required_imports .required_imports
.iter() .iter()
.flat_map(|required_import| { .flat_map(|required_import| {
let Ok(body) = Suite::parse(required_import, "<filename>") else { let Ok(body) = parse_suite(required_import, "<filename>") else {
error!("Failed to parse required import: `{}`", required_import); error!("Failed to parse required import: `{}`", required_import);
return vec![]; return vec![];
}; };

38
crates/ruff/src/rules/mccabe/rules/function_is_too_complex.rs
View file

@ -169,8 +169,8 @@ pub(crate) fn function_is_too_complex(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite;
use ruff_python_parser::Parse; use ruff_python_parser::parse_suite;
use super::get_complexity_number; use super::get_complexity_number;
@ -180,7 +180,7 @@ mod tests {
def trivial(): def trivial():
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -191,7 +191,7 @@ def trivial():
def expr_as_statement(): def expr_as_statement():
0xF00D 0xF00D
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -204,7 +204,7 @@ def sequential(n):
s = k + n s = k + n
return s return s
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -220,7 +220,7 @@ def if_elif_else_dead_path(n):
else: else:
return "smaller than or equal to three" return "smaller than or equal to three"
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -237,7 +237,7 @@ def nested_ifs():
else: else:
return "smaller than or equal to three" return "smaller than or equal to three"
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -249,7 +249,7 @@ def for_loop():
for i in range(10): for i in range(10):
print(i) print(i)
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -263,7 +263,7 @@ def for_else(mylist):
else: else:
print(None) print(None)
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -277,7 +277,7 @@ def recursive(n):
else: else:
return n return n
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -294,7 +294,7 @@ def nested_functions():
a() a()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -312,7 +312,7 @@ def try_else():
else: else:
print(4) print(4)
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 4); assert_eq!(get_complexity_number(&stmts), 4);
Ok(()) Ok(())
} }
@ -329,7 +329,7 @@ def nested_try_finally():
finally: finally:
print(3) print(3)
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -346,7 +346,7 @@ async def foobar(a, b, c):
async for x in a: async for x in a:
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -357,7 +357,7 @@ async def foobar(a, b, c):
def annotated_assign(): def annotated_assign():
x: Any = None x: Any = None
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -393,7 +393,7 @@ class Class:
return ServiceProvider(Logger()) return ServiceProvider(Logger())
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 9); assert_eq!(get_complexity_number(&stmts), 9);
Ok(()) Ok(())
} }
@ -407,7 +407,7 @@ def process_detect_lines():
finally: finally:
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -422,7 +422,7 @@ def process_detect_lines():
if res: if res:
errors.append(f"Non-zero exit code {res}") errors.append(f"Non-zero exit code {res}")
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -435,7 +435,7 @@ def with_lock():
if foo: if foo:
print('bar') print('bar')
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }

5
crates/ruff/src/rules/pylint/rules/too_many_branches.rs
View file

@ -183,13 +183,12 @@ pub(crate) fn too_many_branches(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite; use ruff_python_parser::parse_suite;
use ruff_python_parser::Parse;
use super::num_branches; use super::num_branches;
fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> { fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> {
let branches = Suite::parse(source, "<filename>")?; let branches = parse_suite(source, "<filename>")?;
assert_eq!(num_branches(&branches), expected_num_branches); assert_eq!(num_branches(&branches), expected_num_branches);
Ok(()) Ok(())
} }

5
crates/ruff/src/rules/pylint/rules/too_many_return_statements.rs
View file

@ -98,13 +98,12 @@ pub(crate) fn too_many_return_statements(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite; use ruff_python_parser::parse_suite;
use ruff_python_parser::Parse;
use super::num_returns; use super::num_returns;
fn test_helper(source: &str, expected: usize) -> Result<()> { fn test_helper(source: &str, expected: usize) -> Result<()> {
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_returns(&stmts), expected); assert_eq!(num_returns(&stmts), expected);
Ok(()) Ok(())
} }

39
crates/ruff/src/rules/pylint/rules/too_many_statements.rs
View file

@ -166,8 +166,7 @@ pub(crate) fn too_many_statements(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite; use ruff_python_parser::parse_suite;
use ruff_python_parser::Parse;
use super::num_statements; use super::num_statements;
@ -177,7 +176,7 @@ mod tests {
def f(): def f():
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }
@ -191,7 +190,7 @@ def f():
else: else:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -206,7 +205,7 @@ def f():
if a: if a:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 6); assert_eq!(num_statements(&stmts), 6);
Ok(()) Ok(())
} }
@ -220,7 +219,7 @@ def f():
elif a: elif a:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -238,7 +237,7 @@ def f():
else: else:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 9); assert_eq!(num_statements(&stmts), 9);
Ok(()) Ok(())
} }
@ -267,7 +266,7 @@ async def f():
import time import time
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 19); assert_eq!(num_statements(&stmts), 19);
Ok(()) Ok(())
} }
@ -279,7 +278,7 @@ def f():
for i in range(10): for i in range(10):
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }
@ -293,7 +292,7 @@ def f():
else: else:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 3); assert_eq!(num_statements(&stmts), 3);
Ok(()) Ok(())
} }
@ -308,7 +307,7 @@ def f():
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -326,7 +325,7 @@ def f():
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 3); assert_eq!(num_statements(&stmts), 3);
Ok(()) Ok(())
} }
@ -337,7 +336,7 @@ def f():
def f(): def f():
return return
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 1); assert_eq!(num_statements(&stmts), 1);
Ok(()) Ok(())
} }
@ -353,7 +352,7 @@ def f():
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 6); assert_eq!(num_statements(&stmts), 6);
Ok(()) Ok(())
} }
@ -367,7 +366,7 @@ def f():
except Exception: except Exception:
raise raise
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -383,7 +382,7 @@ def f():
else: else:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 7); assert_eq!(num_statements(&stmts), 7);
Ok(()) Ok(())
} }
@ -401,7 +400,7 @@ def f():
finally: finally:
pass pass
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 10); assert_eq!(num_statements(&stmts), 10);
Ok(()) Ok(())
} }
@ -417,7 +416,7 @@ def f():
except Exception: except Exception:
raise raise
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 8); assert_eq!(num_statements(&stmts), 8);
Ok(()) Ok(())
} }
@ -435,7 +434,7 @@ def f():
finally: finally:
print() print()
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 11); assert_eq!(num_statements(&stmts), 11);
Ok(()) Ok(())
} }
@ -447,7 +446,7 @@ def f():
for i in range(10): for i in range(10):
yield i yield i
"#; "#;
let stmts = Suite::parse(source, "<filename>")?; let stmts = parse_suite(source, "<filename>")?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }

6
crates/ruff_benchmark/benches/parser.rs
View file

@ -4,8 +4,8 @@ use criterion::measurement::WallTime;
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError}; use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor}; use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
use ruff_python_ast::{Stmt, Suite}; use ruff_python_ast::Stmt;
use ruff_python_parser::Parse; use ruff_python_parser::parse_suite;
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
#[global_allocator] #[global_allocator]
@ -66,7 +66,7 @@ fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
&case, &case,
|b, case| { |b, case| {
b.iter(|| { b.iter(|| {
let parsed = Suite::parse(case.code(), case.name()).unwrap(); let parsed = parse_suite(case.code(), case.name()).unwrap();
let mut visitor = CountVisitor { count: 0 }; let mut visitor = CountVisitor { count: 0 };
visitor.visit_body(&parsed); visitor.visit_body(&parsed);

5
crates/ruff_dev/src/print_ast.rs
View file

@ -5,8 +5,7 @@ use std::fs;
use std::path::PathBuf; use std::path::PathBuf;
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::Suite; use ruff_python_parser::parse_suite;
use ruff_python_parser::Parse;
#[derive(clap::Args)] #[derive(clap::Args)]
pub(crate) struct Args { pub(crate) struct Args {
@ -17,7 +16,7 @@ pub(crate) struct Args {
pub(crate) fn main(args: &Args) -> Result<()> { pub(crate) fn main(args: &Args) -> Result<()> {
let contents = fs::read_to_string(&args.file)?; let contents = fs::read_to_string(&args.file)?;
let python_ast = Suite::parse(&contents, &args.file.to_string_lossy())?; let python_ast = parse_suite(&contents, &args.file.to_string_lossy())?;
println!("{python_ast:#?}"); println!("{python_ast:#?}");
Ok(()) Ok(())
} }

8
crates/ruff_python_ast/tests/identifier.rs
View file

@ -1,5 +1,4 @@
use ruff_python_ast::Stmt; use ruff_python_parser::{parse_suite, ParseError};
use ruff_python_parser::{Parse, ParseError};
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
use ruff_python_ast::identifier; use ruff_python_ast::identifier;
@ -13,8 +12,9 @@ else:
pass pass
"# "#
.trim(); .trim();
let stmt = Stmt::parse(contents, "<filename>")?; let stmts = parse_suite(contents, "<filename>")?;
let range = identifier::else_(&stmt, contents).unwrap(); let stmt = stmts.first().unwrap();
let range = identifier::else_(stmt, contents).unwrap();
assert_eq!(&contents[range], "else"); assert_eq!(&contents[range], "else");
assert_eq!( assert_eq!(
range, range,

18
crates/ruff_python_ast/tests/stmt_if.rs
View file

@ -1,6 +1,6 @@
use ruff_python_ast::stmt_if::elif_else_range; use ruff_python_ast::stmt_if::elif_else_range;
use ruff_python_ast::Stmt;
use ruff_python_parser::{Parse, ParseError}; use ruff_python_parser::{parse_suite, ParseError};
use ruff_text_size::TextSize; use ruff_text_size::TextSize;
#[test] #[test]
@ -10,8 +10,11 @@ fn extract_elif_else_range() -> Result<(), ParseError> {
elif b: elif b:
... ...
"; ";
let stmt = Stmt::parse(contents, "<filename>")?; let mut stmts = parse_suite(contents, "<filename>")?;
let stmt = Stmt::as_if_stmt(&stmt).unwrap(); let stmt = stmts
.pop()
.and_then(ruff_python_ast::Stmt::if_stmt)
.unwrap();
let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.start(), TextSize::from(14));
assert_eq!(range.end(), TextSize::from(18)); assert_eq!(range.end(), TextSize::from(18));
@ -21,8 +24,11 @@ elif b:
else: else:
... ...
"; ";
let stmt = Stmt::parse(contents, "<filename>")?; let mut stmts = parse_suite(contents, "<filename>")?;
let stmt = Stmt::as_if_stmt(&stmt).unwrap(); let stmt = stmts
.pop()
.and_then(ruff_python_ast::Stmt::if_stmt)
.unwrap();
let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap(); let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
assert_eq!(range.start(), TextSize::from(14)); assert_eq!(range.start(), TextSize::from(14));
assert_eq!(range.end(), TextSize::from(18)); assert_eq!(range.end(), TextSize::from(18));

12
crates/ruff_python_codegen/src/generator.rs
View file

@ -1503,8 +1503,8 @@ impl<'a> Generator<'a> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use ruff_python_ast::{Mod, ModModule, Stmt}; use ruff_python_ast::{Mod, ModModule};
use ruff_python_parser::{self, Mode, Parse}; use ruff_python_parser::{self, parse_suite, Mode};
use ruff_source_file::LineEnding; use ruff_source_file::LineEnding;
@ -1515,9 +1515,9 @@ mod tests {
let indentation = Indentation::default(); let indentation = Indentation::default();
let quote = Quote::default(); let quote = Quote::default();
let line_ending = LineEnding::default(); let line_ending = LineEnding::default();
let stmt = Stmt::parse(contents, "<filename>").unwrap(); let stmt = parse_suite(contents, "<filename>").unwrap();
let mut generator = Generator::new(&indentation, quote, line_ending); let mut generator = Generator::new(&indentation, quote, line_ending);
generator.unparse_stmt(&stmt); generator.unparse_suite(&stmt);
generator.generate() generator.generate()
} }
@ -1527,9 +1527,9 @@ mod tests {
line_ending: LineEnding, line_ending: LineEnding,
contents: &str, contents: &str,
) -> String { ) -> String {
let stmt = Stmt::parse(contents, "<filename>").unwrap(); let stmt = parse_suite(contents, "<filename>").unwrap();
let mut generator = Generator::new(indentation, quote, line_ending); let mut generator = Generator::new(indentation, quote, line_ending);
generator.unparse_stmt(&stmt); generator.unparse_suite(&stmt);
generator.generate() generator.generate()
} }

5
crates/ruff_python_codegen/src/lib.rs
View file

@ -2,15 +2,14 @@ mod generator;
mod stylist; mod stylist;
pub use generator::Generator; pub use generator::Generator;
use ruff_python_ast::Suite; use ruff_python_parser::{lexer, parse_suite, Mode, ParseError};
use ruff_python_parser::{lexer, Mode, Parse, ParseError};
use ruff_source_file::Locator; use ruff_source_file::Locator;
pub use stylist::{Quote, Stylist}; pub use stylist::{Quote, Stylist};
/// Run round-trip source code generation on a given Python code. /// Run round-trip source code generation on a given Python code.
pub fn round_trip(code: &str, source_path: &str) -> Result<String, ParseError> { pub fn round_trip(code: &str, source_path: &str) -> Result<String, ParseError> {
let locator = Locator::new(code); let locator = Locator::new(code);
let python_ast = Suite::parse(code, source_path)?; let python_ast = parse_suite(code, source_path)?;
let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect(); let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect();
let stylist = Stylist::from_tokens(&tokens, &locator); let stylist = Stylist::from_tokens(&tokens, &locator);
let mut generator: Generator = (&stylist).into(); let mut generator: Generator = (&stylist).into();

6
crates/ruff_python_formatter/src/expression/expr_name.rs
View file

@ -36,13 +36,13 @@ impl NeedsParentheses for ExprName {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use ruff_python_ast::{ModModule, Ranged}; use ruff_python_ast::Ranged;
use ruff_python_parser::Parse; use ruff_python_parser::parse_program;
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
#[test] #[test]
fn name_range_with_comments() { fn name_range_with_comments() {
let source = ModModule::parse("a # comment", "file.py").unwrap(); let source = parse_program("a # comment", "file.py").unwrap();
let expression_statement = source let expression_statement = source
.body .body

6
crates/ruff_python_formatter/src/statement/suite.rs
View file

@ -210,8 +210,8 @@ impl<'ast> IntoFormat<PyFormatContext<'ast>> for Suite {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use ruff_formatter::format; use ruff_formatter::format;
use ruff_python_ast::Suite;
use ruff_python_parser::Parse; use ruff_python_parser::parse_suite;
use crate::comments::Comments; use crate::comments::Comments;
use crate::prelude::*; use crate::prelude::*;
@ -240,7 +240,7 @@ def trailing_func():
pass pass
"#; "#;
let statements = Suite::parse(source, "test.py").unwrap(); let statements = parse_suite(source, "test.py").unwrap();
let context = PyFormatContext::new(PyFormatOptions::default(), source, Comments::default()); let context = PyFormatContext::new(PyFormatOptions::default(), source, Comments::default());

39
crates/ruff_python_parser/src/context.rs
View file

@ -49,132 +49,131 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::Parse; use crate::parser::parse_suite;
use ruff_python_ast as ast;
#[test] #[test]
fn test_assign_name() { fn test_assign_name() {
let source = "x = (1, 2, 3)"; let source = "x = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_tuple() { fn test_assign_tuple() {
let source = "(x, y) = (1, 2, 3)"; let source = "(x, y) = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_list() { fn test_assign_list() {
let source = "[x, y] = (1, 2, 3)"; let source = "[x, y] = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_attribute() { fn test_assign_attribute() {
let source = "x.y = (1, 2, 3)"; let source = "x.y = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_subscript() { fn test_assign_subscript() {
let source = "x[y] = (1, 2, 3)"; let source = "x[y] = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_starred() { fn test_assign_starred() {
let source = "(x, *y) = (1, 2, 3)"; let source = "(x, *y) = (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_for() { fn test_assign_for() {
let source = "for x in (1, 2, 3): pass"; let source = "for x in (1, 2, 3): pass";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_list_comp() { fn test_assign_list_comp() {
let source = "x = [y for y in (1, 2, 3)]"; let source = "x = [y for y in (1, 2, 3)]";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_set_comp() { fn test_assign_set_comp() {
let source = "x = {y for y in (1, 2, 3)}"; let source = "x = {y for y in (1, 2, 3)}";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_with() { fn test_assign_with() {
let source = "with 1 as x: pass"; let source = "with 1 as x: pass";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_named_expr() { fn test_assign_named_expr() {
let source = "if x:= 1: pass"; let source = "if x:= 1: pass";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_ann_assign_name() { fn test_ann_assign_name() {
let source = "x: int = 1"; let source = "x: int = 1";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_name() { fn test_aug_assign_name() {
let source = "x += 1"; let source = "x += 1";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_attribute() { fn test_aug_assign_attribute() {
let source = "x.y += (1, 2, 3)"; let source = "x.y += (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_subscript() { fn test_aug_assign_subscript() {
let source = "x[y] += (1, 2, 3)"; let source = "x[y] += (1, 2, 3)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_name() { fn test_del_name() {
let source = "del x"; let source = "del x";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_attribute() { fn test_del_attribute() {
let source = "del x.y"; let source = "del x.y";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_subscript() { fn test_del_subscript() {
let source = "del x[y]"; let source = "del x[y]";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
} }

8
crates/ruff_python_parser/src/function.rs
View file

@ -133,15 +133,15 @@ const fn is_starred(exp: &ast::Expr) -> bool {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::{Parse, ParseErrorType}; use crate::parser::parse_suite;
use ruff_python_ast::{self as ast}; use crate::ParseErrorType;
macro_rules! function_and_lambda { macro_rules! function_and_lambda {
($($name:ident: $code:expr,)*) => { ($($name:ident: $code:expr,)*) => {
$( $(
#[test] #[test]
fn $name() { fn $name() {
let parse_ast = ast::Suite::parse($code, "<test>"); let parse_ast = crate::parser::parse_suite($code, "<test>");
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
)* )*
@ -172,7 +172,7 @@ mod tests {
} }
fn function_parse_error(src: &str) -> LexicalErrorType { fn function_parse_error(src: &str) -> LexicalErrorType {
let parse_ast = ast::Suite::parse(src, "<test>"); let parse_ast = parse_suite(src, "<test>");
parse_ast parse_ast
.map_err(|e| match e.error { .map_err(|e| match e.error {
ParseErrorType::Lexical(e) => e, ParseErrorType::Lexical(e) => e,

40
crates/ruff_python_parser/src/lib.rs
View file

@ -94,14 +94,13 @@
//! mode or tokenizing the source beforehand: //! mode or tokenizing the source beforehand:
//! //!
//! ``` //! ```
//! use ruff_python_parser::{Parse}; //! use ruff_python_parser::parse_suite;
//! use ruff_python_ast as ast;
//! //!
//! let python_source = r#" //! let python_source = r#"
//! def is_odd(i): //! def is_odd(i):
//! return bool(i & 1) //! return bool(i & 1)
//! "#; //! "#;
//! let ast = ast::Suite::parse(python_source, "<embedded>"); //! let ast = parse_suite(python_source, "<embedded>");
//! //!
//! assert!(ast.is_ok()); //! assert!(ast.is_ok());
//! ``` //! ```
@ -111,11 +110,11 @@
//! [lexer]: crate::lexer //! [lexer]: crate::lexer
use crate::lexer::LexResult; use crate::lexer::LexResult;
pub use parse::Parse; pub use parser::{
pub use parser::{parse, parse_starts_at, parse_tokens, ParseError, ParseErrorType}; parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
#[allow(deprecated)] parse_suite, parse_tokens, ParseError, ParseErrorType,
pub use parser::{parse_expression, parse_expression_starts_at, parse_program}; };
use ruff_python_ast::{CmpOp, Expr, ModModule, Ranged, Suite}; use ruff_python_ast::{CmpOp, Expr, Mod, Ranged, Suite};
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
pub use string::FStringErrorType; pub use string::FStringErrorType;
pub use token::{StringKind, Tok, TokenKind}; pub use token::{StringKind, Tok, TokenKind};
@ -124,7 +123,6 @@ mod function;
// Skip flattening lexer to distinguish from full ruff_python_parser // Skip flattening lexer to distinguish from full ruff_python_parser
mod context; mod context;
pub mod lexer; pub mod lexer;
mod parse;
mod parser; mod parser;
mod soft_keywords; mod soft_keywords;
mod string; mod string;
@ -149,7 +147,10 @@ pub fn parse_program_tokens(
lxr: Vec<LexResult>, lxr: Vec<LexResult>,
source_path: &str, source_path: &str,
) -> anyhow::Result<Suite, ParseError> { ) -> anyhow::Result<Suite, ParseError> {
ModModule::parse_tokens(lxr, source_path).map(|module| module.body) match parse_tokens(lxr, Mode::Module, source_path)? {
Mod::Module(m) => Ok(m.body),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}
} }
/// Return the `Range` of the first `Tok::Colon` token in a `Range`. /// Return the `Range` of the first `Tok::Colon` token in a `Range`.
@ -344,11 +345,10 @@ mod python {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::Parse; use crate::{first_colon_range, locate_cmp_ops, parse_expression, LocatedCmpOp};
use crate::{first_colon_range, locate_cmp_ops, LocatedCmpOp};
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::CmpOp; use ruff_python_ast::CmpOp;
use ruff_python_ast::Expr;
use ruff_text_size::{TextLen, TextRange, TextSize}; use ruff_text_size::{TextLen, TextRange, TextSize};
#[test] #[test]
@ -366,7 +366,7 @@ mod tests {
#[test] #[test]
fn extract_cmp_op_location() -> Result<()> { fn extract_cmp_op_location() -> Result<()> {
let contents = "x == 1"; let contents = "x == 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -376,7 +376,7 @@ mod tests {
); );
let contents = "x != 1"; let contents = "x != 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -386,7 +386,7 @@ mod tests {
); );
let contents = "x is 1"; let contents = "x is 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -396,7 +396,7 @@ mod tests {
); );
let contents = "x is not 1"; let contents = "x is not 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -406,7 +406,7 @@ mod tests {
); );
let contents = "x in 1"; let contents = "x in 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -416,7 +416,7 @@ mod tests {
); );
let contents = "x not in 1"; let contents = "x not in 1";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -426,7 +426,7 @@ mod tests {
); );
let contents = "x != (1 is not 2)"; let contents = "x != (1 is not 2)";
let expr = Expr::parse(contents, "<filename>")?; let expr = parse_expression(contents, "<filename>")?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(

1177
crates/ruff_python_parser/src/parse.rs
View file

File diff suppressed because it is too large Load diff

131
crates/ruff_python_parser/src/parser.rs
View file

@ -18,14 +18,15 @@ use itertools::Itertools;
pub(super) use lalrpop_util::ParseError as LalrpopError; pub(super) use lalrpop_util::ParseError as LalrpopError;
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
use crate::lexer::{lex, lex_starts_at};
use crate::{ use crate::{
lexer::{self, LexResult, LexicalError, LexicalErrorType}, lexer::{self, LexResult, LexicalError, LexicalErrorType},
python, python,
token::Tok, token::Tok,
Mode, Parse, Mode,
}; };
use ruff_python_ast as ast; use ruff_python_ast as ast;
use ruff_python_ast::ModModule; use ruff_python_ast::{Mod, ModModule, Suite};
/// Parse a full Python program usually consisting of multiple lines. /// Parse a full Python program usually consisting of multiple lines.
/// ///
@ -47,9 +48,16 @@ use ruff_python_ast::ModModule;
/// let program = parser::parse_program(source, "<embedded>"); /// let program = parser::parse_program(source, "<embedded>");
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Suite::parse from ruff_python_parser::Parse trait."] pub fn parse_program(source: &str, source_path: &str) -> Result<ModModule, ParseError> {
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> { let lexer = lex(source, Mode::Module);
ModModule::parse(source, source_path).map(|module| module.body) match parse_tokens(lexer, Mode::Module, source_path)? {
Mod::Module(m) => Ok(m),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}
}
pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError> {
parse_program(source, source_path).map(|m| m.body)
} }
/// Parses a single Python expression. /// Parses a single Python expression.
@ -68,9 +76,12 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ///
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Expr::parse from ruff_python_parser::Parse trait."] pub fn parse_expression(source: &str, source_path: &str) -> Result<ast::Expr, ParseError> {
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> { let lexer = lex(source, Mode::Expression);
ast::Expr::parse(source, path) match parse_tokens(lexer, Mode::Expression, source_path)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
} }
/// Parses a Python expression from a given location. /// Parses a Python expression from a given location.
@ -90,13 +101,16 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400)); /// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Expr::parse_starts_at from ruff_python_parser::Parse trait."]
pub fn parse_expression_starts_at( pub fn parse_expression_starts_at(
source: &str, source: &str,
path: &str, source_path: &str,
offset: TextSize, offset: TextSize,
) -> Result<ast::Expr, ParseError> { ) -> Result<ast::Expr, ParseError> {
ast::Expr::parse_starts_at(source, path, offset) let lexer = lex_starts_at(source, Mode::Module, offset);
match parse_tokens(lexer, Mode::Expression, source_path)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
} }
/// Parse the given Python source code using the specified [`Mode`]. /// Parse the given Python source code using the specified [`Mode`].
@ -384,71 +398,69 @@ impl ParseErrorType {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::Parse;
use insta::assert_debug_snapshot; use insta::assert_debug_snapshot;
use ruff_python_ast as ast;
use super::*; use super::*;
#[test] #[test]
fn test_parse_empty() { fn test_parse_empty() {
let parse_ast = ast::Suite::parse("", "<test>").unwrap(); let parse_ast = parse_suite("", "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_string() { fn test_parse_string() {
let source = "'Hello world'"; let source = "'Hello world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string() { fn test_parse_f_string() {
let source = "f'Hello world'"; let source = "f'Hello world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_print_hello() { fn test_parse_print_hello() {
let source = "print('Hello world')"; let source = "print('Hello world')";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_print_2() { fn test_parse_print_2() {
let source = "print('Hello world', 2)"; let source = "print('Hello world', 2)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_kwargs() { fn test_parse_kwargs() {
let source = "my_func('positional', keyword=2)"; let source = "my_func('positional', keyword=2)";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_if_elif_else() { fn test_parse_if_elif_else() {
let source = "if 1: 10\nelif 2: 20\nelse: 30"; let source = "if 1: 10\nelif 2: 20\nelse: 30";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_lambda() { fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_lambda_no_args() { fn test_parse_lambda_no_args() {
let source = "lambda: 1"; let source = "lambda: 1";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -456,7 +468,7 @@ mod tests {
fn test_parse_tuples() { fn test_parse_tuples() {
let source = "a, b = 4, 5"; let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -468,7 +480,7 @@ class Foo(A, B):
def method_with_default(self, arg='default'): def method_with_default(self, arg='default'):
pass pass
"; ";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -499,7 +511,7 @@ class Foo[**P](): ...
class Foo[X, Y: str, *U, **P](): class Foo[X, Y: str, *U, **P]():
pass pass
"; ";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
fn test_parse_function_definition() { fn test_parse_function_definition() {
@ -525,69 +537,69 @@ def func[**P](*args: P.args, **kwargs: P.kwargs):
def func[T, U: str, *Ts, **P](): def func[T, U: str, *Ts, **P]():
pass pass
"; ";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
fn test_parse_dict_comprehension() { fn test_parse_dict_comprehension() {
let source = "{x1: x2 for y in z}"; let source = "{x1: x2 for y in z}";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_list_comprehension() { fn test_parse_list_comprehension() {
let source = "[x for y in z]"; let source = "[x for y in z]";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_double_list_comprehension() { fn test_parse_double_list_comprehension() {
let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]"; let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_generator_comprehension() { fn test_parse_generator_comprehension() {
let source = "(x for y in z)"; let source = "(x for y in z)";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_named_expression_generator_comprehension() { fn test_parse_named_expression_generator_comprehension() {
let source = "(x := y + 1 for y in z)"; let source = "(x := y + 1 for y in z)";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_if_else_generator_comprehension() { fn test_parse_if_else_generator_comprehension() {
let source = "(x if y else y for y in z)"; let source = "(x if y else y for y in z)";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_bool_op_or() { fn test_parse_bool_op_or() {
let source = "x or y"; let source = "x or y";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_bool_op_and() { fn test_parse_bool_op_and() {
let source = "x and y"; let source = "x and y";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_slice() { fn test_slice() {
let source = "x[1:2:3]"; let source = "x[1:2:3]";
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -621,7 +633,7 @@ with (0 as a,): pass
with (0 as a, 1 as b): pass with (0 as a, 1 as b): pass
with (0 as a, 1 as b,): pass with (0 as a, 1 as b,): pass
"; ";
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -644,7 +656,7 @@ with (0 as a, 1 as b,): pass
"with a := 0 as x: pass", "with a := 0 as x: pass",
"with (a := 0 as x): pass", "with (a := 0 as x): pass",
] { ] {
assert!(ast::Suite::parse(source, "<test>").is_err()); assert!(parse_suite(source, "<test>").is_err());
} }
} }
@ -656,7 +668,7 @@ array[0, *indexes, -1] = array_slice
array[*indexes_to_select, *indexes_to_select] array[*indexes_to_select, *indexes_to_select]
array[3:5, *indexes_to_select] array[3:5, *indexes_to_select]
"; ";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -669,13 +681,13 @@ array[3:5, *indexes_to_select]
("OFFSET %d" % offset) if offset else None, ("OFFSET %d" % offset) if offset else None,
) )
)"#; )"#;
let parse_ast = ast::Expr::parse(source, "<test>").unwrap(); let parse_ast = parse_expression(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_try() { fn test_try() {
let parse_ast = ast::Suite::parse( let parse_ast = parse_suite(
r#"try: r#"try:
raise ValueError(1) raise ValueError(1)
except TypeError as e: except TypeError as e:
@ -690,7 +702,7 @@ except OSError as e:
#[test] #[test]
fn test_try_star() { fn test_try_star() {
let parse_ast = ast::Suite::parse( let parse_ast = parse_suite(
r#"try: r#"try:
raise ExceptionGroup("eg", raise ExceptionGroup("eg",
[ValueError(1), TypeError(2), OSError(3), OSError(4)]) [ValueError(1), TypeError(2), OSError(3), OSError(4)])
@ -706,7 +718,7 @@ except* OSError as e:
#[test] #[test]
fn test_dict_unpacking() { fn test_dict_unpacking() {
let parse_ast = ast::Expr::parse(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap(); let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -758,7 +770,7 @@ type X \
type X[T] \ type X[T] \
= T = T
"#; "#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -795,7 +807,7 @@ type = 1
type = x = 1 type = x = 1
x = type = 1 x = type = 1
"#; "#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -820,7 +832,7 @@ x = 10000
x = 133333 x = 133333
"#; "#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -846,7 +858,7 @@ if 10 .real:
y = 100[no] y = 100[no]
y = 100(no) y = 100(no)
"#; "#;
assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -874,7 +886,7 @@ match match:
match = lambda query: query == event match = lambda query: query == event
print(match(12)) print(match(12))
"#; "#;
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
} }
#[test] #[test]
@ -1044,13 +1056,13 @@ match w := x,:
case y as v,: case y as v,:
z = 0 z = 0
"#; "#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_match() { fn test_match() {
let parse_ast = ast::Suite::parse( let parse_ast = parse_suite(
r#" r#"
match {"test": 1}: match {"test": 1}:
case { case {
@ -1080,7 +1092,7 @@ match x:
#[test] #[test]
fn test_variadic_generics() { fn test_variadic_generics() {
let parse_ast = ast::Suite::parse( let parse_ast = parse_suite(
r#" r#"
def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
"#, "#,
@ -1090,26 +1102,9 @@ def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test]
fn test_parse_constant() {
use num_traits::ToPrimitive;
let c = ast::Constant::parse_without_path("'string'").unwrap();
assert_eq!(c.str().unwrap(), "string");
let c = ast::Constant::parse_without_path("10").unwrap();
assert_eq!(c.int().unwrap().to_i32().unwrap(), 10);
}
#[test]
fn test_parse_identifier() {
let i = ast::Identifier::parse_without_path("test").unwrap();
assert_eq!(i.as_str(), "test");
}
#[test] #[test]
fn decorator_ranges() { fn decorator_ranges() {
let parse_ast = ast::Suite::parse( let parse_ast = parse_suite(
r#" r#"
@my_decorator @my_decorator
def test(): def test():

54
crates/ruff_python_parser/src/string.rs
View file

@ -11,9 +11,9 @@ use ruff_text_size::{TextLen, TextRange, TextSize};
// we have to do the parsing here, manually. // we have to do the parsing here, manually.
use crate::{ use crate::{
lexer::{LexicalError, LexicalErrorType}, lexer::{LexicalError, LexicalErrorType},
parse_expression_starts_at,
parser::{ParseError, ParseErrorType}, parser::{ParseError, ParseErrorType},
token::{StringKind, Tok}, token::{StringKind, Tok},
Parse,
}; };
// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
@ -550,7 +550,7 @@ impl<'a> StringParser<'a> {
fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> { fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> {
let fstring_body = format!("({source})"); let fstring_body = format!("({source})");
ast::Expr::parse_starts_at(&fstring_body, "<fstring>", location) parse_expression_starts_at(&fstring_body, "<fstring>", location)
} }
fn parse_string( fn parse_string(
@ -785,10 +785,8 @@ impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalEr
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::Parse;
use ruff_python_ast as ast;
use super::*; use super::*;
use crate::parser::parse_suite;
fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> { fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
StringParser::new(source, StringKind::FString, false, TextSize::default()).parse() StringParser::new(source, StringKind::FString, false, TextSize::default()).parse()
@ -926,63 +924,63 @@ mod tests {
#[test] #[test]
fn test_parse_string_concat() { fn test_parse_string_concat() {
let source = "'Hello ' 'world'"; let source = "'Hello ' 'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_string_concat_1() { fn test_parse_u_string_concat_1() {
let source = "'Hello ' u'world'"; let source = "'Hello ' u'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_string_concat_2() { fn test_parse_u_string_concat_2() {
let source = "u'Hello ' 'world'"; let source = "u'Hello ' 'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_1() { fn test_parse_f_string_concat_1() {
let source = "'Hello ' f'world'"; let source = "'Hello ' f'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_2() { fn test_parse_f_string_concat_2() {
let source = "'Hello ' f'world'"; let source = "'Hello ' f'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_3() { fn test_parse_f_string_concat_3() {
let source = "'Hello ' f'world{\"!\"}'"; let source = "'Hello ' f'world{\"!\"}'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_f_string_concat_1() { fn test_parse_u_f_string_concat_1() {
let source = "u'Hello ' f'world'"; let source = "u'Hello ' f'world'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_f_string_concat_2() { fn test_parse_u_f_string_concat_2() {
let source = "u'Hello ' f'world' '!'"; let source = "u'Hello ' f'world' '!'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_string_triple_quotes_with_kind() { fn test_parse_string_triple_quotes_with_kind() {
let source = "u'''Hello, world!'''"; let source = "u'''Hello, world!'''";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -990,7 +988,7 @@ mod tests {
fn test_single_quoted_byte() { fn test_single_quoted_byte() {
// single quote // single quote
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -998,7 +996,7 @@ mod tests {
fn test_double_quoted_byte() { fn test_double_quoted_byte() {
// double quote // double quote
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -1006,42 +1004,42 @@ mod tests {
fn test_escape_char_in_byte_literal() { fn test_escape_char_in_byte_literal() {
// backslash does not escape // backslash does not escape
let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_byte_literal_1() { fn test_raw_byte_literal_1() {
let source = r"rb'\x1z'"; let source = r"rb'\x1z'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_byte_literal_2() { fn test_raw_byte_literal_2() {
let source = r"rb'\\'"; let source = r"rb'\\'";
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_escape_octet() { fn test_escape_octet() {
let source = r##"b'\43a\4\1234'"##; let source = r##"b'\43a\4\1234'"##;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_escaped_newline() { fn test_fstring_escaped_newline() {
let source = r#"f"\n{x}""#; let source = r#"f"\n{x}""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_constant_range() { fn test_fstring_constant_range() {
let source = r#"f"aaa{bbb}ccc{ddd}eee""#; let source = r#"f"aaa{bbb}ccc{ddd}eee""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -1049,28 +1047,28 @@ mod tests {
fn test_fstring_unescaped_newline() { fn test_fstring_unescaped_newline() {
let source = r#"f""" let source = r#"f"""
{x}""""#; {x}""""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_escaped_character() { fn test_fstring_escaped_character() {
let source = r#"f"\\{x}""#; let source = r#"f"\\{x}""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_fstring() { fn test_raw_fstring() {
let source = r#"rf"{x}""#; let source = r#"rf"{x}""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_triple_quoted_raw_fstring() { fn test_triple_quoted_raw_fstring() {
let source = r#"rf"""{x}""""#; let source = r#"rf"""{x}""""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -1078,7 +1076,7 @@ mod tests {
fn test_fstring_line_continuation() { fn test_fstring_line_continuation() {
let source = r#"rf"\ let source = r#"rf"\
{x}""#; {x}""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap(); let parse_ast = parse_suite(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -1088,7 +1086,7 @@ mod tests {
#[test] #[test]
fn $name() { fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias); let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = ast::Suite::parse(&source, "<test>").unwrap(); let parse_ast = parse_suite(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
)* )*

6
crates/ruff_python_parser/src/typing.rs
View file

@ -1,4 +1,4 @@
use crate::Parse; use crate::{parse_expression, parse_expression_starts_at};
use anyhow::Result; use anyhow::Result;
use ruff_python_ast::relocate::relocate_expr; use ruff_python_ast::relocate::relocate_expr;
use ruff_python_ast::str; use ruff_python_ast::str;
@ -33,7 +33,7 @@ pub fn parse_type_annotation(
// isn't the case, e.g., for implicit concatenations, or for annotations that contain // isn't the case, e.g., for implicit concatenations, or for annotations that contain
// escaped quotes. // escaped quotes.
let leading_quote = str::leading_quote(expression).unwrap(); let leading_quote = str::leading_quote(expression).unwrap();
let expr = Expr::parse_starts_at( let expr = parse_expression_starts_at(
value, value,
"<filename>", "<filename>",
range.start() + leading_quote.text_len(), range.start() + leading_quote.text_len(),
@ -41,7 +41,7 @@ pub fn parse_type_annotation(
Ok((expr, AnnotationKind::Simple)) Ok((expr, AnnotationKind::Simple))
} else { } else {
// Otherwise, consider this a "complex" annotation. // Otherwise, consider this a "complex" annotation.
let mut expr = Expr::parse(value, "<filename>")?; let mut expr = parse_expression(value, "<filename>")?;
relocate_expr(&mut expr, range); relocate_expr(&mut expr, range);
Ok((expr, AnnotationKind::Complex)) Ok((expr, AnnotationKind::Complex))
} }

14
crates/ruff_python_trivia/src/whitespace.rs
View file

@ -83,32 +83,32 @@ impl PythonWhitespace for str {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::has_trailing_content; use crate::has_trailing_content;
use ruff_python_ast::{Ranged, Suite}; use ruff_python_ast::Ranged;
use ruff_python_parser::{Parse, ParseError}; use ruff_python_parser::{parse_suite, ParseError};
use ruff_source_file::Locator; use ruff_source_file::Locator;
#[test] #[test]
fn trailing_content() -> Result<(), ParseError> { fn trailing_content() -> Result<(), ParseError> {
let contents = "x = 1"; let contents = "x = 1";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1; y = 2"; let contents = "x = 1; y = 2";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(has_trailing_content(stmt.end(), &locator)); assert!(has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 "; let contents = "x = 1 ";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 # Comment"; let contents = "x = 1 # Comment";
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
@ -118,7 +118,7 @@ x = 1
y = 2 y = 2
"# "#
.trim(); .trim();
let program = Suite::parse(contents, "<filename>")?; let program = parse_suite(contents, "<filename>")?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));

5
fuzz/fuzz_targets/ruff_parse_simple.rs
View file

@ -4,9 +4,8 @@
#![no_main] #![no_main]
use libfuzzer_sys::{fuzz_target, Corpus}; use libfuzzer_sys::{fuzz_target, Corpus};
use ruff_python_ast::Suite;
use ruff_python_codegen::{Generator, Stylist}; use ruff_python_codegen::{Generator, Stylist};
use ruff_python_parser::{lexer, Mode, Parse, ParseError}; use ruff_python_parser::{lexer, parse_suite, Mode, ParseError};
use ruff_source_file::Locator; use ruff_source_file::Locator;
fn do_fuzz(case: &[u8]) -> Corpus { fn do_fuzz(case: &[u8]) -> Corpus {
@ -16,7 +15,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
// just round-trip it once to trigger both parse and unparse // just round-trip it once to trigger both parse and unparse
let locator = Locator::new(code); let locator = Locator::new(code);
let python_ast = match Suite::parse(code, "fuzzed-source.py") { let python_ast = match parse_suite(code, "fuzzed-source.py") {
Ok(stmts) => stmts, Ok(stmts) => stmts,
Err(ParseError { offset, .. }) => { Err(ParseError { offset, .. }) => {
let offset = offset.to_usize(); let offset = offset.to_usize();