mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 18:28:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Remove source path from parser errors (#9322)

Browse source 
## Summary

I always found it odd that we had to pass this in, since it's really
higher-level context for the error. The awkwardness is further evidenced
by the fact that we pass in fake values everywhere (even outside of
tests). The source path isn't actually used to display the error; it's
only accessed elsewhere to _re-display_ the error in certain cases. This
PR modifies to instead pass the path directly in those cases.
This commit is contained in:
Charlie Marsh 2023-12-30 16:33:05 -04:00 • committed by GitHub
parent eb9a1bc5f1
commit e80260a3c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 623 additions and 714 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/ruff_benchmark/benches/formatter.rs
View file

@ -65,7 +65,7 @@ fn benchmark_formatter(criterion: &mut Criterion) {
let comment_ranges = comment_ranges.finish();
// Parse the AST.
let module = parse_tokens(tokens, case.code(), Mode::Module, "<filename>")
let module = parse_tokens(tokens, case.code(), Mode::Module)
.expect("Input to be a valid python program");
b.iter(|| {

3
crates/ruff_benchmark/benches/linter.rs
View file

@ -58,8 +58,7 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
let tokens = lexer::lex(case.code(), Mode::Module).collect::<Vec<_>>();
// Parse the source.
let ast =
parse_program_tokens(tokens.clone(), case.code(), case.name(), false).unwrap();
let ast = parse_program_tokens(tokens.clone(), case.code(), false).unwrap();
b.iter(|| {
let path = case.path();

2
crates/ruff_benchmark/benches/parser.rs
View file

@ -60,7 +60,7 @@ fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
&case,
|b, case| {
b.iter(|| {
let parsed = parse_suite(case.code(), case.name()).unwrap();
let parsed = parse_suite(case.code()).unwrap();
let mut visitor = CountVisitor { count: 0 };
visitor.visit_body(&parsed);

1
crates/ruff_cli/src/diagnostics.rs
View file

@ -366,6 +366,7 @@ pub(crate) fn lint_path(
&LineIndex::from_source_text(source_kind.source_code())
),
&source_kind,
path,
)
);
}

6
crates/ruff_dev/src/print_ast.rs
View file

@ -24,11 +24,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
args.file.display()
)
})?;
let python_ast = parse(
source_kind.source_code(),
source_type.as_mode(),
&args.file.to_string_lossy(),
)?;
let python_ast = parse(source_kind.source_code(), source_type.as_mode())?;
println!("{python_ast:#?}");
Ok(())
}

2
crates/ruff_dev/src/round_trip.rs
View file

@ -22,7 +22,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
println!("{}", ruff_notebook::round_trip(path)?);
} else {
let contents = fs::read_to_string(&args.file)?;
println!("{}", round_trip(&contents, &args.file.to_string_lossy())?);
println!("{}", round_trip(&contents)?);
}
Ok(())
}

8
crates/ruff_linter/src/fix/edits.rs
View file

@ -397,13 +397,13 @@ mod tests {
#[test]
fn find_semicolon() -> Result<()> {
let contents = "x = 1";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert_eq!(trailing_semicolon(stmt.end(), &locator), None);
let contents = "x = 1; y = 1";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert_eq!(
@ -412,7 +412,7 @@ mod tests {
);
let contents = "x = 1 ; y = 1";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert_eq!(
@ -425,7 +425,7 @@ x = 1 \
; y = 1
"
.trim();
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert_eq!(

2
crates/ruff_linter/src/importer/insertion.rs
View file

@ -333,7 +333,7 @@ mod tests {
#[test]
fn start_of_file() -> Result<()> {
fn insert(contents: &str) -> Result<Insertion> {
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let tokens: Vec<LexResult> = ruff_python_parser::tokenize(contents, Mode::Module);
let locator = Locator::new(contents);
let stylist = Stylist::from_tokens(&tokens, &locator);

6
crates/ruff_linter/src/linter.rs
View file

@ -145,7 +145,7 @@ pub fn check_path(
.any(|rule_code| rule_code.lint_source().is_imports());
if use_ast || use_imports || use_doc_lines {
// Parse, if the AST wasn't pre-provided provided.
match tokens.into_ast_source(source_kind, source_type, path) {
match tokens.into_ast_source(source_kind, source_type) {
Ok(python_ast) => {
let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets);
if use_ast {
@ -339,7 +339,7 @@ pub fn add_noqa_to_path(
if let Some(error) = error {
error!(
"{}",
DisplayParseError::new(error, locator.to_source_code(), source_kind)
DisplayParseError::new(error, locator.to_source_code(), source_kind, path)
);
}
@ -687,13 +687,11 @@ impl<'a> TokenSource<'a> {
self,
source_kind: &SourceKind,
source_type: PySourceType,
path: &Path,
) -> Result<AstSource<'a>, ParseError> {
match self {
Self::Tokens(tokens) => Ok(AstSource::Ast(ruff_python_parser::parse_program_tokens(
tokens,
source_kind.source_code(),
&path.to_string_lossy(),
source_type.is_ipynb(),
)?)),
Self::Precomputed { ast, .. } => Ok(AstSource::Precomputed(ast)),

5
crates/ruff_linter/src/logging.rs
View file

@ -140,6 +140,7 @@ pub struct DisplayParseError<'a> {
error: ParseError,
source_code: SourceCode<'a, 'a>,
source_kind: &'a SourceKind,
path: &'a Path,
}
impl<'a> DisplayParseError<'a> {
@ -147,11 +148,13 @@ impl<'a> DisplayParseError<'a> {
error: ParseError,
source_code: SourceCode<'a, 'a>,
source_kind: &'a SourceKind,
path: &'a Path,
) -> Self {
Self {
error,
source_code,
source_kind,
path,
}
}
}
@ -162,7 +165,7 @@ impl Display for DisplayParseError<'_> {
f,
"{header} {path}{colon}",
header = "Failed to parse".bold(),
path = fs::relativize_path(Path::new(&self.error.source_path)).bold(),
path = fs::relativize_path(self.path).bold(),
colon = ":".cyan(),
)?;

2
crates/ruff_linter/src/rules/eradicate/detection.rs
View file

@ -74,7 +74,7 @@ pub(crate) fn comment_contains_code(line: &str, task_tags: &[String]) -> bool {
}
// Finally, compile the source code.
parse_suite(line, "<filename>").is_ok()
parse_suite(line).is_ok()
}
#[cfg(test)]

2
crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs
View file

@ -135,7 +135,7 @@ pub(crate) fn add_required_imports(
.required_imports
.iter()
.flat_map(|required_import| {
let Ok(body) = parse_suite(required_import, "<filename>") else {
let Ok(body) = parse_suite(required_import) else {
error!("Failed to parse required import: `{}`", required_import);
return vec![];
};

34
crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs
View file

@ -170,7 +170,7 @@ mod tests {
def trivial():
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -181,7 +181,7 @@ def trivial():
def expr_as_statement():
0xF00D
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -194,7 +194,7 @@ def sequential(n):
s = k + n
return s
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -210,7 +210,7 @@ def if_elif_else_dead_path(n):
else:
return "smaller than or equal to three"
"#;
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3);
Ok(())
}
@ -227,7 +227,7 @@ def nested_ifs():
else:
return "smaller than or equal to three"
"#;
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3);
Ok(())
}
@ -239,7 +239,7 @@ def for_loop():
for i in range(10):
print(i)
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2);
Ok(())
}
@ -253,7 +253,7 @@ def for_else(mylist):
else:
print(None)
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2);
Ok(())
}
@ -267,7 +267,7 @@ def recursive(n):
else:
return n
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2);
Ok(())
}
@ -284,7 +284,7 @@ def nested_functions():
a()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3);
Ok(())
}
@ -302,7 +302,7 @@ def try_else():
else:
print(4)
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 4);
Ok(())
}
@ -319,7 +319,7 @@ def nested_try_finally():
finally:
print(3)
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -336,7 +336,7 @@ async def foobar(a, b, c):
async for x in a:
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3);
Ok(())
}
@ -347,7 +347,7 @@ async def foobar(a, b, c):
def annotated_assign():
x: Any = None
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -383,7 +383,7 @@ class Class:
return ServiceProvider(Logger())
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 9);
Ok(())
}
@ -397,7 +397,7 @@ def process_detect_lines():
finally:
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1);
Ok(())
}
@ -412,7 +412,7 @@ def process_detect_lines():
if res:
errors.append(f"Non-zero exit code {res}")
"#;
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2);
Ok(())
}
@ -425,7 +425,7 @@ def with_lock():
if foo:
print('bar')
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2);
Ok(())
}

2
crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs
View file

@ -180,7 +180,7 @@ mod tests {
use super::num_branches;
fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> {
let branches = parse_suite(source, "<filename>")?;
let branches = parse_suite(source)?;
assert_eq!(num_branches(&branches), expected_num_branches);
Ok(())
}

2
crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs
View file

@ -103,7 +103,7 @@ mod tests {
use super::num_returns;
fn test_helper(source: &str, expected: usize) -> Result<()> {
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_returns(&stmts), expected);
Ok(())
}

36
crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs
View file

@ -167,7 +167,7 @@ mod tests {
def f():
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2);
Ok(())
}
@ -181,7 +181,7 @@ def f():
else:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5);
Ok(())
}
@ -196,7 +196,7 @@ def f():
if a:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 6);
Ok(())
}
@ -210,7 +210,7 @@ def f():
elif a:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5);
Ok(())
}
@ -228,7 +228,7 @@ def f():
else:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 9);
Ok(())
}
@ -257,7 +257,7 @@ async def f():
import time
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 19);
Ok(())
}
@ -269,7 +269,7 @@ def f():
for i in range(10):
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2);
Ok(())
}
@ -283,7 +283,7 @@ def f():
else:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 3);
Ok(())
}
@ -298,7 +298,7 @@ def f():
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5);
Ok(())
}
@ -316,7 +316,7 @@ def f():
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 3);
Ok(())
}
@ -327,7 +327,7 @@ def f():
def f():
return
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 1);
Ok(())
}
@ -343,7 +343,7 @@ def f():
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 6);
Ok(())
}
@ -357,7 +357,7 @@ def f():
except Exception:
raise
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5);
Ok(())
}
@ -373,7 +373,7 @@ def f():
else:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 7);
Ok(())
}
@ -391,7 +391,7 @@ def f():
finally:
pass
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 10);
Ok(())
}
@ -407,7 +407,7 @@ def f():
except Exception:
raise
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 8);
Ok(())
}
@ -425,7 +425,7 @@ def f():
finally:
print()
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 11);
Ok(())
}
@ -437,7 +437,7 @@ def f():
for i in range(10):
yield i
";
let stmts = parse_suite(source, "<filename>")?;
let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2);
Ok(())
}

2
crates/ruff_linter/src/rules/ruff/rules/unreachable.rs
View file

@ -1053,7 +1053,7 @@ mod tests {
fn control_flow_graph(filename: &str) {
let path = PathBuf::from_iter(["resources/test/fixtures/control-flow-graph", filename]);
let source = fs::read_to_string(path).expect("failed to read file");
let stmts = parse(&source, Mode::Module, filename)
let stmts = parse(&source, Mode::Module)
.unwrap_or_else(|err| panic!("failed to parse source: '{source}': {err}"))
.expect_module()
.body;

2
crates/ruff_python_ast/tests/identifier.rs
View file

@ -12,7 +12,7 @@ else:
pass
"
.trim();
let stmts = parse_suite(contents, "<filename>")?;
let stmts = parse_suite(contents)?;
let stmt = stmts.first().unwrap();
let range = identifier::else_(stmt, contents).unwrap();
assert_eq!(&contents[range], "else");

16
crates/ruff_python_ast/tests/parenthesize.rs
View file

@ -6,7 +6,7 @@ use ruff_text_size::TextRange;
#[test]
fn test_parenthesized_name() {
let source_code = r"(x) + 1";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
@ -23,7 +23,7 @@ fn test_parenthesized_name() {
#[test]
fn test_non_parenthesized_name() {
let source_code = r"x + 1";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
@ -40,7 +40,7 @@ fn test_non_parenthesized_name() {
#[test]
fn test_parenthesized_argument() {
let source_code = r"f((a))";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments;
@ -58,7 +58,7 @@ fn test_parenthesized_argument() {
#[test]
fn test_non_parenthesized_argument() {
let source_code = r"f(a)";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments;
@ -76,7 +76,7 @@ fn test_non_parenthesized_argument() {
#[test]
fn test_parenthesized_tuple_member() {
let source_code = r"(a, (b))";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap();
@ -93,7 +93,7 @@ fn test_parenthesized_tuple_member() {
#[test]
fn test_non_parenthesized_tuple_member() {
let source_code = r"(a, b)";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap();
@ -110,7 +110,7 @@ fn test_non_parenthesized_tuple_member() {
#[test]
fn test_twice_parenthesized_name() {
let source_code = r"((x)) + 1";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
@ -127,7 +127,7 @@ fn test_twice_parenthesized_name() {
#[test]
fn test_twice_parenthesized_argument() {
let source_code = r"f(((a + 1)))";
let expr = parse_expression(source_code, "<filename>").unwrap();
let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments;

2
crates/ruff_python_ast/tests/preorder.rs
View file

@ -149,7 +149,7 @@ fn f_strings() {
fn trace_preorder_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, source, Mode::Module, "test.py").unwrap();
let parsed = parse_tokens(tokens, source, Mode::Module).unwrap();
let mut visitor = RecordVisitor::default();
visitor.visit_mod(&parsed);

4
crates/ruff_python_ast/tests/stmt_if.rs
View file

@ -10,7 +10,7 @@ fn extract_elif_else_range() -> Result<(), ParseError> {
elif b:
...
";
let mut stmts = parse_suite(contents, "<filename>")?;
let mut stmts = parse_suite(contents)?;
let stmt = stmts
.pop()
.and_then(ruff_python_ast::Stmt::if_stmt)
@ -24,7 +24,7 @@ elif b:
else:
...
";
let mut stmts = parse_suite(contents, "<filename>")?;
let mut stmts = parse_suite(contents)?;
let stmt = stmts
.pop()
.and_then(ruff_python_ast::Stmt::if_stmt)

2
crates/ruff_python_ast/tests/visitor.rs
View file

@ -160,7 +160,7 @@ fn f_strings() {
fn trace_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, source, Mode::Module, "test.py").unwrap();
let parsed = parse_tokens(tokens, source, Mode::Module).unwrap();
let mut visitor = RecordVisitor::default();
walk_module(&mut visitor, &parsed);

6
crates/ruff_python_codegen/src/generator.rs
View file

@ -1416,7 +1416,7 @@ mod tests {
let indentation = Indentation::default();
let quote = Quote::default();
let line_ending = LineEnding::default();
let stmt = parse_suite(contents, "<filename>").unwrap();
let stmt = parse_suite(contents).unwrap();
let mut generator = Generator::new(&indentation, quote, line_ending);
generator.unparse_suite(&stmt);
generator.generate()
@ -1428,7 +1428,7 @@ mod tests {
line_ending: LineEnding,
contents: &str,
) -> String {
let stmt = parse_suite(contents, "<filename>").unwrap();
let stmt = parse_suite(contents).unwrap();
let mut generator = Generator::new(indentation, quote, line_ending);
generator.unparse_suite(&stmt);
generator.generate()
@ -1438,7 +1438,7 @@ mod tests {
let indentation = Indentation::default();
let quote = Quote::default();
let line_ending = LineEnding::default();
let ast = ruff_python_parser::parse(contents, Mode::Ipython, "<filename>").unwrap();
let ast = ruff_python_parser::parse(contents, Mode::Ipython).unwrap();
let Mod::Module(ModModule { body, .. }) = ast else {
panic!("Source code didn't return ModModule")
};

4
crates/ruff_python_codegen/src/lib.rs
View file

@ -7,9 +7,9 @@ use ruff_source_file::Locator;
pub use stylist::{Quote, Stylist};
/// Run round-trip source code generation on a given Python code.
pub fn round_trip(code: &str, source_path: &str) -> Result<String, ParseError> {
pub fn round_trip(code: &str) -> Result<String, ParseError> {
let locator = Locator::new(code);
let python_ast = parse_suite(code, source_path)?;
let python_ast = parse_suite(code)?;
let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect();
let stylist = Stylist::from_tokens(&tokens, &locator);
let mut generator: Generator = (&stylist).into();

4
crates/ruff_python_formatter/src/cli.rs
View file

@ -50,8 +50,8 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
.map_err(|err| format_err!("Source contains syntax errors {err:?}"))?;
// Parse the AST.
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>")
.context("Syntax error in input")?;
let module =
parse_ok_tokens(tokens, source, source_type.as_mode()).context("Syntax error in input")?;
let options = PyFormatOptions::from_extension(source_path)
.with_preview(if cli.preview {

2
crates/ruff_python_formatter/src/comments/mod.rs
View file

@ -568,7 +568,7 @@ mod tests {
let source_type = PySourceType::Python;
let (tokens, comment_ranges) =
tokens_and_ranges(source, source_type).expect("Expect source to be valid Python");
let parsed = parse_ok_tokens(tokens, source, source_type.as_mode(), "test.py")
let parsed = parse_ok_tokens(tokens, source, source_type.as_mode())
.expect("Expect source to be valid Python");
CommentsTestCase {

2
crates/ruff_python_formatter/src/expression/expr_name.rs
View file

@ -52,7 +52,7 @@ mod tests {
#[test]
fn name_range_with_comments() {
let source = parse_program("a # comment", "file.py").unwrap();
let source = parse_program("a # comment").unwrap();
let expression_statement = source
.body

2
crates/ruff_python_formatter/src/expression/parentheses.rs
View file

@ -447,7 +447,7 @@ mod tests {
#[test]
fn test_has_parentheses() {
let expression = r#"(b().c("")).d()"#;
let expr = parse_expression(expression, "<filename>").unwrap();
let expr = parse_expression(expression).unwrap();
assert!(!is_expression_parenthesized(
ExpressionRef::from(&expr),
&CommentRanges::default(),

4
crates/ruff_python_formatter/src/lib.rs
View file

@ -137,7 +137,7 @@ pub fn format_module_source(
) -> Result<Printed, FormatModuleError> {
let source_type = options.source_type();
let (tokens, comment_ranges) = tokens_and_ranges(source, source_type)?;
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>")?;
let module = parse_ok_tokens(tokens, source, source_type.as_mode())?;
let formatted = format_module_ast(&module, &comment_ranges, source, options)?;
Ok(formatted.print()?)
}
@ -225,7 +225,7 @@ def main() -> None:
// Parse the AST.
let source_path = "code_inline.py";
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), source_path).unwrap();
let module = parse_ok_tokens(tokens, source, source_type.as_mode()).unwrap();
let options = PyFormatOptions::from_extension(Path::new(source_path));
let formatted = format_module_ast(&module, &comment_ranges, source, options).unwrap();

2
crates/ruff_python_formatter/src/statement/suite.rs
View file

@ -746,7 +746,7 @@ def trailing_func():
pass
";
let statements = parse_suite(source, "test.py").unwrap();
let statements = parse_suite(source).unwrap();
let comment_ranges = CommentRanges::default();
let context = PyFormatContext::new(

9
crates/ruff_python_formatter/src/string/docstring.rs
View file

@ -524,11 +524,7 @@ impl<'ast, 'buf, 'fmt, 'src> DocstringLinePrinter<'ast, 'buf, 'fmt, 'src> {
std::format!(r#""""{}""""#, printed.as_code())
}
};
let result = ruff_python_parser::parse(
&wrapped,
self.f.options().source_type().as_mode(),
"<filename>",
);
let result = ruff_python_parser::parse(&wrapped, self.f.options().source_type().as_mode());
// If the resulting code is not valid, then reset and pass through
// the docstring lines as-is.
if result.is_err() {
@ -1523,8 +1519,7 @@ fn docstring_format_source(
let source_type = options.source_type();
let (tokens, comment_ranges) = ruff_python_index::tokens_and_ranges(source, source_type)?;
let module =
ruff_python_parser::parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>")?;
let module = ruff_python_parser::parse_ok_tokens(tokens, source, source_type.as_mode())?;
let source_code = ruff_formatter::SourceCode::new(source);
let comments = crate::Comments::from_ast(&module, source_code, &comment_ranges);
let locator = Locator::new(source);

14
crates/ruff_python_formatter/tests/fixtures.rs
View file

@ -300,22 +300,14 @@ fn ensure_unchanged_ast(
let source_type = options.source_type();
// Parse the unformatted code.
let mut unformatted_ast = parse(
unformatted_code,
source_type.as_mode(),
&input_path.to_string_lossy(),
)
let mut unformatted_ast = parse(unformatted_code, source_type.as_mode())
.expect("Unformatted code to be valid syntax");
Normalizer.visit_module(&mut unformatted_ast);
let unformatted_ast = ComparableMod::from(&unformatted_ast);
// Parse the formatted code.
let mut formatted_ast = parse(
formatted_code,
source_type.as_mode(),
&input_path.to_string_lossy(),
)
.expect("Formatted code to be valid syntax");
let mut formatted_ast =
parse(formatted_code, source_type.as_mode()).expect("Formatted code to be valid syntax");
Normalizer.visit_module(&mut formatted_ast);
let formatted_ast = ComparableMod::from(&formatted_ast);

36
crates/ruff_python_parser/src/context.rs
View file

@ -54,126 +54,126 @@ mod tests {
#[test]
fn test_assign_name() {
let source = "x = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_tuple() {
let source = "(x, y) = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_list() {
let source = "[x, y] = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_attribute() {
let source = "x.y = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_subscript() {
let source = "x[y] = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_starred() {
let source = "(x, *y) = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_for() {
let source = "for x in (1, 2, 3): pass";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_list_comp() {
let source = "x = [y for y in (1, 2, 3)]";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_set_comp() {
let source = "x = {y for y in (1, 2, 3)}";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_with() {
let source = "with 1 as x: pass";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_named_expr() {
let source = "if x:= 1: pass";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_ann_assign_name() {
let source = "x: int = 1";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_name() {
let source = "x += 1";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_attribute() {
let source = "x.y += (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_subscript() {
let source = "x[y] += (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_name() {
let source = "del x";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_attribute() {
let source = "del x.y";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_subscript() {
let source = "del x[y]";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

4
crates/ruff_python_parser/src/function.rs
View file

@ -148,7 +148,7 @@ mod tests {
$(
#[test]
fn $name() {
let parse_ast = crate::parser::parse_suite($code, "<test>");
let parse_ast = crate::parser::parse_suite($code, );
insta::assert_debug_snapshot!(parse_ast);
}
)*
@ -179,7 +179,7 @@ mod tests {
}
fn function_parse_error(src: &str) -> LexicalErrorType {
let parse_ast = parse_suite(src, "<test>");
let parse_ast = parse_suite(src);
parse_ast
.map_err(|e| match e.error {
ParseErrorType::Lexical(e) => e,

387
crates/ruff_python_parser/src/invalid.rs
View file

@ -100,343 +100,322 @@ mod tests {
// Regression test: https://github.com/astral-sh/ruff/issues/6895
#[test]
fn err_literal_assignment() {
let ast = parse_suite(r"5 = 3", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"5 = 3");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
// This test previously passed before the assignment operator checking
// above, but we include it here for good measure.
#[test]
fn err_assignment_expr() {
let ast = parse_suite(r"(5 := 3)", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(5 := 3)");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: UnrecognizedToken(
ColonEqual,
None,
),
offset: 3,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_literal_augment_assignment() {
let ast = parse_suite(r"5 += 3", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"5 += 3");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_literal_annotation_assignment() {
let ast = parse_suite(r"(5): int = 3", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(5): int = 3");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
// Now we exhaustively test all possible cases where assignment can fail.
#[test]
fn err_bool_op() {
let ast = parse_suite(r"x or y = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"x or y = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_named_expr() {
let ast = parse_suite(r"(x := 5) = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(x := 5) = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_bin_op() {
let ast = parse_suite(r"x + y = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"x + y = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_unary_op() {
let ast = parse_suite(r"-x = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"-x = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_lambda() {
let ast = parse_suite(r"(lambda _: 1) = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(lambda _: 1) = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_if_exp() {
let ast = parse_suite(r"a if b else c = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"a if b else c = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_dict() {
let ast = parse_suite(r"{'a':5} = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"{'a':5} = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_set() {
let ast = parse_suite(r"{a} = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"{a} = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_list_comp() {
let ast = parse_suite(r"[x for x in xs] = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"[x for x in xs] = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_set_comp() {
let ast = parse_suite(r"{x for x in xs} = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"{x for x in xs} = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_dict_comp() {
let ast = parse_suite(r"{x: x*2 for x in xs} = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"{x: x*2 for x in xs} = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_generator_exp() {
let ast = parse_suite(r"(x for x in xs) = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(x for x in xs) = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_await() {
let ast = parse_suite(r"await x = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"await x = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_yield() {
let ast = parse_suite(r"(yield x) = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(yield x) = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_yield_from() {
let ast = parse_suite(r"(yield from xs) = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(yield from xs) = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_compare() {
let ast = parse_suite(r"a < b < c = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"a < b < c = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_call() {
let ast = parse_suite(r"foo() = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"foo() = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
@ -452,194 +431,182 @@ Err(
// is coupled.
//
// See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
let ast = parse_suite(r#"f"{quux}" = 42"#, "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r#"f"{quux}" = 42"#);
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_fstring() {
let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#, "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#);
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_string_literal() {
let ast = parse_suite(r#""foo" = 42"#, "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r#""foo" = 42"#);
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_bytes_literal() {
let ast = parse_suite(r#"b"foo" = 42"#, "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r#"b"foo" = 42"#);
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_number_literal() {
let ast = parse_suite(r"123 = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"123 = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_boolean_literal() {
let ast = parse_suite(r"True = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"True = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_none_literal() {
let ast = parse_suite(r"None = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"None = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_ellipsis_literal() {
let ast = parse_suite(r"... = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"... = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 0,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_starred() {
let ast = parse_suite(r"*foo() = 42", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"*foo() = 42");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 1,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_list() {
let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 4,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_list_nested() {
let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 11,
source_path: "<test>",
},
)
"#);
)
"###);
}
#[test]
fn err_tuple() {
let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)", "<test>");
insta::assert_debug_snapshot!(ast, @r#"
Err(
let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)");
insta::assert_debug_snapshot!(ast, @r###"
Err(
ParseError {
error: Lexical(
AssignmentError,
),
offset: 4,
source_path: "<test>",
},
)
"#);
)
"###);
}
// This last group of tests checks that assignments we expect to be parsed
@ -647,25 +614,25 @@ Err(
#[test]
fn ok_starred() {
let ast = parse_suite(r"*foo = 42", "<test>");
let ast = parse_suite(r"*foo = 42");
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_list() {
let ast = parse_suite(r"[x, y, z] = [1, 2, 3]", "<test>");
let ast = parse_suite(r"[x, y, z] = [1, 2, 3]");
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_tuple() {
let ast = parse_suite(r"(x, y, z) = (1, 2, 3)", "<test>");
let ast = parse_suite(r"(x, y, z) = (1, 2, 3)");
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_subscript_normal() {
let ast = parse_suite(r"x[0] = 42", "<test>");
let ast = parse_suite(r"x[0] = 42");
insta::assert_debug_snapshot!(ast);
}
@ -673,13 +640,13 @@ Err(
// doesn't fail parsing.
#[test]
fn ok_subscript_weird() {
let ast = parse_suite(r"5[0] = 42", "<test>");
let ast = parse_suite(r"5[0] = 42");
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_slice_normal() {
let ast = parse_suite(r"x[1:2] = [42]", "<test>");
let ast = parse_suite(r"x[1:2] = [42]");
insta::assert_debug_snapshot!(ast);
}
@ -687,13 +654,13 @@ Err(
// doesn't fail parsing.
#[test]
fn ok_slice_weird() {
let ast = parse_suite(r"5[1:2] = [42]", "<test>");
let ast = parse_suite(r"5[1:2] = [42]");
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_attribute_normal() {
let ast = parse_suite(r"foo.bar = 42", "<test>");
let ast = parse_suite(r"foo.bar = 42");
insta::assert_debug_snapshot!(ast);
}
@ -701,13 +668,13 @@ Err(
// it doesn't fail parsing.
#[test]
fn ok_attribute_weird() {
let ast = parse_suite(r#""foo".y = 42"#, "<test>");
let ast = parse_suite(r#""foo".y = 42"#);
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_name() {
let ast = parse_suite(r"foo = 42", "<test>");
let ast = parse_suite(r"foo = 42");
insta::assert_debug_snapshot!(ast);
}
@ -720,13 +687,13 @@ Err(
let src = r"!foo = 42";
let tokens = crate::lexer::lex(src, Mode::Ipython);
let ast = crate::parse_tokens(tokens, src, Mode::Ipython, "<test>");
let ast = crate::parse_tokens(tokens, src, Mode::Ipython);
insta::assert_debug_snapshot!(ast);
}
#[test]
fn ok_assignment_expr() {
let ast = parse_suite(r"(x := 5)", "<test>");
let ast = parse_suite(r"(x := 5)");
insta::assert_debug_snapshot!(ast);
}
}

21
crates/ruff_python_parser/src/lib.rs
View file

@ -85,7 +85,7 @@
//! return bool(i & 1)
//! "#;
//! let tokens = lex(python_source, Mode::Module);
//! let ast = parse_tokens(tokens, python_source, Mode::Module, "<embedded>");
//! let ast = parse_tokens(tokens, python_source, Mode::Module);
//!
//! assert!(ast.is_ok());
//! ```
@ -100,7 +100,7 @@
//! def is_odd(i):
//! return bool(i & 1)
//! "#;
//! let ast = parse_suite(python_source, "<embedded>");
//! let ast = parse_suite(python_source);
//!
//! assert!(ast.is_ok());
//! ```
@ -148,7 +148,6 @@ pub fn tokenize(contents: &str, mode: Mode) -> Vec<LexResult> {
pub fn parse_program_tokens(
lxr: Vec<LexResult>,
source: &str,
source_path: &str,
is_jupyter_notebook: bool,
) -> anyhow::Result<Suite, ParseError> {
let mode = if is_jupyter_notebook {
@ -156,7 +155,7 @@ pub fn parse_program_tokens(
} else {
Mode::Module
};
match parse_tokens(lxr, source, mode, source_path)? {
match parse_tokens(lxr, source, mode)? {
Mod::Module(m) => Ok(m.body),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}
@ -379,7 +378,7 @@ mod tests {
#[test]
fn extract_cmp_op_location() -> Result<()> {
let contents = "x == 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -389,7 +388,7 @@ mod tests {
);
let contents = "x != 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -399,7 +398,7 @@ mod tests {
);
let contents = "x is 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -409,7 +408,7 @@ mod tests {
);
let contents = "x is not 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -419,7 +418,7 @@ mod tests {
);
let contents = "x in 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -429,7 +428,7 @@ mod tests {
);
let contents = "x not in 1";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(
@ -439,7 +438,7 @@ mod tests {
);
let contents = "x != (1 is not 2)";
let expr = parse_expression(contents, "<filename>")?;
let expr = parse_expression(contents)?;
assert_eq!(
locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new(

170
crates/ruff_python_parser/src/parser.rs
View file

@ -12,8 +12,6 @@
//! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
//! [`Mode`]: crate::mode
use std::{fmt, iter};
use itertools::Itertools;
pub(super) use lalrpop_util::ParseError as LalrpopError;
use ruff_text_size::{Ranged, TextRange, TextSize};
@ -51,19 +49,19 @@ use ruff_python_ast::{
///
/// print(foo())
/// "#;
/// let program = parser::parse_program(source, "<embedded>");
/// let program = parser::parse_program(source);
/// assert!(program.is_ok());
/// ```
pub fn parse_program(source: &str, source_path: &str) -> Result<ModModule, ParseError> {
pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
let lexer = lex(source, Mode::Module);
match parse_tokens(lexer, source, Mode::Module, source_path)? {
match parse_tokens(lexer, source, Mode::Module)? {
Mod::Module(m) => Ok(m),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
}
}
pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError> {
parse_program(source, source_path).map(|m| m.body)
pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
parse_program(source).map(|m| m.body)
}
/// Parses a single Python expression.
@ -77,14 +75,14 @@ pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError>
///
/// ```
/// use ruff_python_parser as parser;
/// let expr = parser::parse_expression("1 + 2", "<embedded>");
/// let expr = parser::parse_expression("1 + 2");
///
/// assert!(expr.is_ok());
///
/// ```
pub fn parse_expression(source: &str, source_path: &str) -> Result<Expr, ParseError> {
pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
let lexer = lex(source, Mode::Expression);
match parse_tokens(lexer, source, Mode::Expression, source_path)? {
match parse_tokens(lexer, source, Mode::Expression)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
@ -104,16 +102,12 @@ pub fn parse_expression(source: &str, source_path: &str) -> Result<Expr, ParseEr
/// use ruff_python_parser::{parse_expression_starts_at};
/// # use ruff_text_size::TextSize;
///
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// let expr = parse_expression_starts_at("1 + 2", TextSize::from(400));
/// assert!(expr.is_ok());
/// ```
pub fn parse_expression_starts_at(
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Expr, ParseError> {
pub fn parse_expression_starts_at(source: &str, offset: TextSize) -> Result<Expr, ParseError> {
let lexer = lex_starts_at(source, Mode::Module, offset);
match parse_tokens(lexer, source, Mode::Expression, source_path)? {
match parse_tokens(lexer, source, Mode::Expression)? {
Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
}
@ -133,7 +127,7 @@ pub fn parse_expression_starts_at(
/// ```
/// use ruff_python_parser::{Mode, parse};
///
/// let expr = parse("1 + 2", Mode::Expression, "<embedded>");
/// let expr = parse("1 + 2", Mode::Expression);
/// assert!(expr.is_ok());
/// ```
///
@ -148,7 +142,7 @@ pub fn parse_expression_starts_at(
/// def greet(self):
/// print("Hello, world!")
/// "#;
/// let program = parse(source, Mode::Module, "<embedded>");
/// let program = parse(source, Mode::Module);
/// assert!(program.is_ok());
/// ```
///
@ -162,11 +156,11 @@ pub fn parse_expression_starts_at(
/// ?str.replace
/// !ls
/// "#;
/// let program = parse(source, Mode::Ipython, "<embedded>");
/// let program = parse(source, Mode::Ipython);
/// assert!(program.is_ok());
/// ```
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<Mod, ParseError> {
parse_starts_at(source, mode, source_path, TextSize::default())
pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
parse_starts_at(source, mode, TextSize::default())
}
/// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
@ -189,17 +183,12 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<Mod, ParseEr
///
/// print(fib(42))
/// "#;
/// let program = parse_starts_at(source, Mode::Module, "<embedded>", TextSize::from(0));
/// let program = parse_starts_at(source, Mode::Module, TextSize::from(0));
/// assert!(program.is_ok());
/// ```
pub fn parse_starts_at(
source: &str,
mode: Mode,
source_path: &str,
offset: TextSize,
) -> Result<Mod, ParseError> {
pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod, ParseError> {
let lxr = lexer::lex_starts_at(source, mode, offset);
parse_tokens(lxr, source, mode, source_path)
parse_tokens(lxr, source, mode)
}
/// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
@ -215,14 +204,13 @@ pub fn parse_starts_at(
/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
///
/// let source = "1 + 2";
/// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression, "<embedded>");
/// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression);
/// assert!(expr.is_ok());
/// ```
pub fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
let lxr = lxr.into_iter();
@ -230,7 +218,6 @@ pub fn parse_tokens(
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
source,
mode,
source_path,
)
}
@ -239,35 +226,33 @@ pub fn parse_ok_tokens(
lxr: impl IntoIterator<Item = Spanned>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
let lxr = lxr
.into_iter()
.filter(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
let marker_token = (Tok::start_marker(mode), TextRange::default());
let lexer = iter::once(marker_token)
let lexer = std::iter::once(marker_token)
.chain(lxr)
.map(|(t, range)| (range.start(), t, range.end()));
python::TopParser::new()
.parse(source, mode, lexer)
.map_err(|e| parse_error_from_lalrpop(e, source_path))
.map_err(parse_error_from_lalrpop)
}
fn parse_filtered_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source: &str,
mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> {
let marker_token = (Tok::start_marker(mode), TextRange::default());
let lexer = iter::once(Ok(marker_token)).chain(lxr);
let lexer = std::iter::once(Ok(marker_token)).chain(lxr);
python::TopParser::new()
.parse(
source,
mode,
lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
)
.map_err(|e| parse_error_from_lalrpop(e, source_path))
.map_err(parse_error_from_lalrpop)
}
/// Represents represent errors that occur during parsing and are
@ -277,7 +262,6 @@ fn parse_filtered_tokens(
pub struct ParseError {
pub error: ParseErrorType,
pub offset: TextSize,
pub source_path: String,
}
impl std::ops::Deref for ParseError {
@ -294,7 +278,7 @@ impl std::error::Error for ParseError {
}
}
impl fmt::Display for ParseError {
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
@ -324,28 +308,20 @@ pub enum ParseErrorType {
impl std::error::Error for ParseErrorType {}
// Convert `lalrpop_util::ParseError` to our internal type
fn parse_error_from_lalrpop(
err: LalrpopError<TextSize, Tok, LexicalError>,
source_path: &str,
) -> ParseError {
let source_path = source_path.to_owned();
fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> ParseError {
match err {
// TODO: Are there cases where this isn't an EOF?
LalrpopError::InvalidToken { location } => ParseError {
error: ParseErrorType::Eof,
offset: location,
source_path,
},
LalrpopError::ExtraToken { token } => ParseError {
error: ParseErrorType::ExtraToken(token.1),
offset: token.0,
source_path,
},
LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error),
offset: error.location,
source_path,
},
LalrpopError::UnrecognizedToken { token, expected } => {
// Hacky, but it's how CPython does it. See PyParser_AddToken,
@ -354,7 +330,6 @@ fn parse_error_from_lalrpop(
ParseError {
error: ParseErrorType::UnrecognizedToken(token.1, expected),
offset: token.0,
source_path,
}
}
LalrpopError::UnrecognizedEof { location, expected } => {
@ -364,13 +339,11 @@ fn parse_error_from_lalrpop(
ParseError {
error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
offset: location,
source_path,
}
} else {
ParseError {
error: ParseErrorType::Eof,
offset: location,
source_path,
}
}
}
@ -629,63 +602,63 @@ mod tests {
#[test]
fn test_parse_empty() {
let parse_ast = parse_suite("", "<test>").unwrap();
let parse_ast = parse_suite("").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string() {
let source = "'Hello world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string() {
let source = "f'Hello world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_hello() {
let source = "print('Hello world')";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_2() {
let source = "print('Hello world', 2)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_kwargs() {
let source = "my_func('positional', keyword=2)";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_elif_else() {
let source = "if 1: 10\nelif 2: 20\nelse: 30";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_lambda_no_args() {
let source = "lambda: 1";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -693,7 +666,7 @@ mod tests {
fn test_parse_tuples() {
let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -705,7 +678,7 @@ class Foo(A, B):
def method_with_default(self, arg='default'):
pass
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -736,7 +709,7 @@ class Foo[**P](): ...
class Foo[X, Y: str, *U, **P]():
pass
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
fn test_parse_function_definition() {
@ -762,76 +735,76 @@ def func[**P](*args: P.args, **kwargs: P.kwargs):
def func[T, U: str, *Ts, **P]():
pass
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
fn test_parse_dict_comprehension() {
let source = "{x1: x2 for y in z}";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_list_comprehension() {
let source = "[x for y in z]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_double_list_comprehension() {
let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_generator_comprehension() {
let source = "(x for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_named_expression_generator_comprehension() {
let source = "(x := y + 1 for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_else_generator_comprehension() {
let source = "(x if y else y for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_or() {
let source = "x or y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_and() {
let source = "x and y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_slice() {
let source = "x[1:2:3]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_named_expression() {
let source = "(x := ( y * z ))";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -865,7 +838,7 @@ with (0 as a,): pass
with (0 as a, 1 as b): pass
with (0 as a, 1 as b,): pass
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -889,7 +862,7 @@ with (yield from a): pass
with ((yield)): pass
with ((yield from a)): pass
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -912,7 +885,7 @@ with ((yield from a)): pass
"with a := 0 as x: pass",
"with (a := 0 as x): pass",
] {
assert!(parse_suite(source, "<test>").is_err());
assert!(parse_suite(source).is_err());
}
}
@ -924,7 +897,7 @@ array[0, *indexes, -1] = array_slice
array[*indexes_to_select, *indexes_to_select]
array[3:5, *indexes_to_select]
";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -937,7 +910,7 @@ array[3:5, *indexes_to_select]
("OFFSET %d" % offset) if offset else None,
)
)"#;
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -950,7 +923,6 @@ except TypeError as e:
print(f'caught {type(e)}')
except OSError as e:
print(f'caught {type(e)}')",
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -966,7 +938,6 @@ except* TypeError as e:
print(f'caught {type(e)} with nested {e.exceptions}')
except* OSError as e:
print(f'caught {type(e)} with nested {e.exceptions}')"#,
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -974,7 +945,7 @@ except* OSError as e:
#[test]
fn test_dict_unpacking() {
let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -982,8 +953,8 @@ except* OSError as e:
fn test_modes() {
let source = "a[0][1][2][3][4]";
assert!(parse(source, Mode::Expression, "<embedded>").is_ok());
assert!(parse(source, Mode::Module, "<embedded>").is_ok());
assert!(parse(source, Mode::Expression).is_ok());
assert!(parse(source, Mode::Module).is_ok());
}
#[test]
@ -1030,7 +1001,7 @@ type X[T] \
type X = int; type X = str; type X = type
class X: type X = int
"#;
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -1068,13 +1039,13 @@ type = x = 1
x = type = 1
lambda x: type
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
fn test_invalid_type() {
assert!(parse_suite("a: type X = int", "<test>").is_err());
assert!(parse_suite("lambda: type X = int", "<test>").is_err());
assert!(parse_suite("a: type X = int").is_err());
assert!(parse_suite("lambda: type X = int").is_err());
}
#[test]
@ -1099,7 +1070,7 @@ x = 10000
x = 133333
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -1125,7 +1096,7 @@ if 10 .real:
y = 100[no]
y = 100(no)
";
assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -1153,7 +1124,7 @@ match match:
match = lambda query: query == event
print(match(12))
";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(source).unwrap());
}
#[test]
@ -1323,7 +1294,7 @@ match w := x,:
case y as v,:
z = 0
"#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1360,7 +1331,6 @@ match x, y,:
case z:
pass
"#,
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1375,7 +1345,6 @@ match x:
case f"{y}":
pass
"#,
"<test>",
)
.err();
assert!(
@ -1390,7 +1359,6 @@ match x:
r"
def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
",
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1409,7 +1377,6 @@ class Abcd:
pass
"
.trim(),
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1486,7 +1453,6 @@ foo.bar[0].baz[2].egg??
"
.trim(),
Mode::Ipython,
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1500,7 +1466,7 @@ a = 1
"
.trim();
let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default());
let parse_err = parse_tokens(lxr, source, Mode::Module, "<test>").unwrap_err();
let parse_err = parse_tokens(lxr, source, Mode::Module).unwrap_err();
assert_eq!(
parse_err.to_string(),
"IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6"
@ -1540,7 +1506,6 @@ f"""{
}"""
"#
.trim(),
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1556,7 +1521,6 @@ u"foo" f"{bar}" "baz" " some"
u"foo" f"bar {baz} really" u"bar" "no"
"#
.trim(),
"<test>",
)
.unwrap();
insta::assert_debug_snapshot!(parse_ast);
@ -1565,7 +1529,7 @@ u"foo" f"bar {baz} really" u"bar" "no"
#[test]
fn test_unicode_aliases() {
// https://github.com/RustPython/RustPython/issues/4566
let parse_ast = parse_suite(r#"x = "\N{BACKSPACE}another cool trick""#, "<test>").unwrap();
let parse_ast = parse_suite(r#"x = "\N{BACKSPACE}another cool trick""#).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

84
crates/ruff_python_parser/src/string.rs
View file

@ -489,7 +489,7 @@ mod tests {
fn string_parser_escaped_eol(eol: &str) -> Suite {
let source = format!(r"'text \{eol}more text'");
parse_suite(&source, "<test>").unwrap()
parse_suite(&source).unwrap()
}
#[test]
@ -513,7 +513,7 @@ mod tests {
#[test]
fn test_parse_fstring() {
let source = r#"f"{a}{ b }{{foo}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -521,7 +521,7 @@ mod tests {
#[test]
fn test_parse_fstring_nested_spec() {
let source = r#"f"{foo:{spec}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -529,20 +529,20 @@ mod tests {
#[test]
fn test_parse_fstring_not_nested_spec() {
let source = r#"f"{foo:spec}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_empty_fstring() {
insta::assert_debug_snapshot!(parse_suite(r#"f"""#, "<test>").unwrap());
insta::assert_debug_snapshot!(parse_suite(r#"f"""#,).unwrap());
}
#[test]
fn test_fstring_parse_self_documenting_base() {
let source = r#"f"{user=}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -550,7 +550,7 @@ mod tests {
#[test]
fn test_fstring_parse_self_documenting_base_more() {
let source = r#"f"mix {user=} with text and {second=}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -558,13 +558,13 @@ mod tests {
#[test]
fn test_fstring_parse_self_documenting_format() {
let source = r#"f"{user=:>10}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
fn parse_fstring_error(source: &str) -> FStringErrorType {
parse_suite(source, "<test>")
parse_suite(source)
.map_err(|e| match e.error {
ParseErrorType::Lexical(LexicalErrorType::FStringError(e)) => e,
e => unreachable!("Expected FStringError: {:?}", e),
@ -585,111 +585,111 @@ mod tests {
parse_fstring_error("f'{lambda x: {x}}'"),
LambdaWithoutParentheses
);
assert!(parse_suite(r#"f"{class}""#, "<test>").is_err());
assert!(parse_suite(r#"f"{class}""#,).is_err());
}
#[test]
fn test_parse_fstring_not_equals() {
let source = r#"f"{1 != 2}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_fstring_equals() {
let source = r#"f"{42 == 42}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_fstring_self_doc_prec_space() {
let source = r#"f"{x =}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_fstring_self_doc_trailing_space() {
let source = r#"f"{x= }""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_fstring_yield_expr() {
let source = r#"f"{yield}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string_concat() {
let source = "'Hello ' 'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_1() {
let source = "'Hello ' u'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_2() {
let source = "u'Hello ' 'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_1() {
let source = "'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_2() {
let source = "'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_3() {
let source = "'Hello ' f'world{\"!\"}'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_4() {
let source = "'Hello ' f'world{\"!\"}' 'again!'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_1() {
let source = "u'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_2() {
let source = "u'Hello ' f'world' '!'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string_triple_quotes_with_kind() {
let source = "u'''Hello, world!'''";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -697,7 +697,7 @@ mod tests {
fn test_single_quoted_byte() {
// single quote
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -705,7 +705,7 @@ mod tests {
fn test_double_quoted_byte() {
// double quote
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -713,42 +713,42 @@ mod tests {
fn test_escape_char_in_byte_literal() {
// backslash does not escape
let source = r#"b"omkmok\Xaa""#; // spell-checker:ignore omkmok
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_1() {
let source = r"rb'\x1z'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_2() {
let source = r"rb'\\'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_escape_octet() {
let source = r"b'\43a\4\1234'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_newline() {
let source = r#"f"\n{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_constant_range() {
let source = r#"f"aaa{bbb}ccc{ddd}eee""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -756,28 +756,28 @@ mod tests {
fn test_fstring_unescaped_newline() {
let source = r#"f"""
{x}""""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_character() {
let source = r#"f"\\{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_fstring() {
let source = r#"rf"{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_triple_quoted_raw_fstring() {
let source = r#"rf"""{x}""""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -785,14 +785,14 @@ mod tests {
fn test_fstring_line_continuation() {
let source = r#"rf"\
{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_fstring_nested_string_spec() {
let source = r#"f"{foo:{''}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -800,7 +800,7 @@ mod tests {
#[test]
fn test_parse_fstring_nested_concatenation_string_spec() {
let source = r#"f"{foo:{'' ''}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -809,7 +809,7 @@ mod tests {
#[test]
fn test_dont_panic_on_8_in_octal_escape() {
let source = r"bold = '\038[1m'";
let parse_ast = parse_suite(source, "<test>").unwrap();
let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -820,7 +820,7 @@ mod tests {
#[test]
fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = parse_suite(&source, "<test>").unwrap();
let parse_ast = parse_suite(&source).unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
)*

8
crates/ruff_python_parser/src/typing.rs
View file

@ -33,15 +33,11 @@ pub fn parse_type_annotation(
// isn't the case, e.g., for implicit concatenations, or for annotations that contain
// escaped quotes.
let leading_quote = str::leading_quote(expression).unwrap();
let expr = parse_expression_starts_at(
value,
"<filename>",
range.start() + leading_quote.text_len(),
)?;
let expr = parse_expression_starts_at(value, range.start() + leading_quote.text_len())?;
Ok((expr, AnnotationKind::Simple))
} else {
// Otherwise, consider this a "complex" annotation.
let mut expr = parse_expression(value, "<filename>")?;
let mut expr = parse_expression(value)?;
relocate_expr(&mut expr, range);
Ok((expr, AnnotationKind::Complex))
}

2
crates/ruff_python_semantic/src/analyze/type_inference.rs
View file

@ -436,7 +436,7 @@ mod tests {
use crate::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType};
fn parse(expression: &str) -> Expr {
parse_expression(expression, "").unwrap()
parse_expression(expression).unwrap()
}
#[test]

10
crates/ruff_python_trivia/src/whitespace.rs
View file

@ -91,25 +91,25 @@ mod tests {
#[test]
fn trailing_content() -> Result<(), ParseError> {
let contents = "x = 1";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1; y = 2";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert!(has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 ";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 # Comment";
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator));
@ -119,7 +119,7 @@ x = 1
y = 2
"
.trim();
let program = parse_suite(contents, "<filename>")?;
let program = parse_suite(contents)?;
let stmt = program.first().unwrap();
let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator));

2
crates/ruff_shrinking/src/main.rs
View file

@ -320,7 +320,7 @@ fn minimization_step(
last_strategy_and_idx: Option<(&'static dyn Strategy, usize)>,
) -> Result<Option<(&'static dyn Strategy, usize, String)>> {
let tokens = ruff_python_parser::tokenize(input, Mode::Module);
let ast = ruff_python_parser::parse_program_tokens(tokens, input, "input.py", false)
let ast = ruff_python_parser::parse_program_tokens(tokens, input, false)
.context("not valid python")?;
// Try the last succeeding strategy first, skipping all that failed last time

12
crates/ruff_wasm/src/lib.rs
View file

@ -248,7 +248,7 @@ impl Workspace {
/// Parses the content and returns its AST
pub fn parse(&self, contents: &str) -> Result<String, Error> {
let parsed = ruff_python_parser::parse(contents, Mode::Module, ".").map_err(into_error)?;
let parsed = ruff_python_parser::parse(contents, Mode::Module).map_err(into_error)?;
Ok(format!("{parsed:#?}"))
}
@ -271,20 +271,20 @@ struct ParsedModule<'a> {
}
impl<'a> ParsedModule<'a> {
fn from_source(source: &'a str) -> Result<Self, Error> {
let tokens: Vec<_> = ruff_python_parser::lexer::lex(source, Mode::Module).collect();
fn from_source(source_code: &'a str) -> Result<Self, Error> {
let tokens: Vec<_> = ruff_python_parser::lexer::lex(source_code, Mode::Module).collect();
let mut comment_ranges = CommentRangesBuilder::default();
for (token, range) in tokens.iter().flatten() {
comment_ranges.visit_token(token, *range);
}
let comment_ranges = comment_ranges.finish();
let module = parse_tokens(tokens, source, Mode::Module, ".").map_err(into_error)?;
let module = parse_tokens(tokens, source_code, Mode::Module).map_err(into_error)?;
Ok(Self {
source_code: source,
comment_ranges,
source_code,
module,
comment_ranges,
})
}

7
fuzz/fuzz_targets/ruff_parse_idempotency.rs
View file

@ -13,9 +13,9 @@ fn do_fuzz(case: &[u8]) -> Corpus {
};
// round trip it once to get a formatted version
if let Ok(first) = round_trip(code, "fuzzed-source.py") {
if let Ok(first) = round_trip(code) {
// round trip it a second time to get a case to compare against
if let Ok(second) = round_trip(&first, "fuzzed-source.py") {
if let Ok(second) = round_trip(&first) {
if cfg!(feature = "full-idempotency") {
// potentially, we don't want to test for full idempotency, but just for unsteady states
// enable the "full-idempotency" feature when fuzzing for full idempotency
@ -31,8 +31,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
} else if first != second {
// by the third time we've round-tripped it, we shouldn't be introducing any more
// changes; if we do, then it's likely that we're in an unsteady parsing state
let third = round_trip(&second, "fuzzed-source.py")
.expect("Couldn't round-trip the processed source.");
let third = round_trip(&second).expect("Couldn't round-trip the processed source.");
let diff = TextDiff::from_lines(&second, &third)
.unified_diff()
.header("Parsed twice", "Parsed three times")

2
fuzz/fuzz_targets/ruff_parse_simple.rs
View file

@ -15,7 +15,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
// just round-trip it once to trigger both parse and unparse
let locator = Locator::new(code);
let python_ast = match parse_suite(code, "fuzzed-source.py") {
let python_ast = match parse_suite(code) {
Ok(stmts) => stmts,
Err(ParseError { offset, .. }) => {
let offset = offset.to_usize();