language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-26 11:59:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Remove source path from parser errors (#9322)

Browse source 
## Summary

I always found it odd that we had to pass this in, since it's really
higher-level context for the error. The awkwardness is further evidenced
by the fact that we pass in fake values everywhere (even outside of
tests). The source path isn't actually used to display the error; it's
only accessed elsewhere to _re-display_ the error in certain cases. This
PR modifies to instead pass the path directly in those cases.
This commit is contained in:
Charlie Marsh 2023-12-30 16:33:05 -04:00 • committed by GitHub
parent eb9a1bc5f1
commit e80260a3c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 623 additions and 714 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/ruff_benchmark/benches/formatter.rs
View file

@ -65,7 +65,7 @@ fn benchmark_formatter(criterion: &mut Criterion) {
let comment_ranges = comment_ranges.finish(); let comment_ranges = comment_ranges.finish();
// Parse the AST. // Parse the AST.
let module = parse_tokens(tokens, case.code(), Mode::Module, "<filename>") let module = parse_tokens(tokens, case.code(), Mode::Module)
.expect("Input to be a valid python program"); .expect("Input to be a valid python program");
b.iter(|| { b.iter(|| {

3
crates/ruff_benchmark/benches/linter.rs
View file

@ -58,8 +58,7 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
let tokens = lexer::lex(case.code(), Mode::Module).collect::<Vec<_>>(); let tokens = lexer::lex(case.code(), Mode::Module).collect::<Vec<_>>();
// Parse the source. // Parse the source.
let ast = let ast = parse_program_tokens(tokens.clone(), case.code(), false).unwrap();
parse_program_tokens(tokens.clone(), case.code(), case.name(), false).unwrap();
b.iter(|| { b.iter(|| {
let path = case.path(); let path = case.path();

2
crates/ruff_benchmark/benches/parser.rs
View file

@ -60,7 +60,7 @@ fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
&case, &case,
|b, case| { |b, case| {
b.iter(|| { b.iter(|| {
let parsed = parse_suite(case.code(), case.name()).unwrap(); let parsed = parse_suite(case.code()).unwrap();
let mut visitor = CountVisitor { count: 0 }; let mut visitor = CountVisitor { count: 0 };
visitor.visit_body(&parsed); visitor.visit_body(&parsed);

1
crates/ruff_cli/src/diagnostics.rs
View file

@ -366,6 +366,7 @@ pub(crate) fn lint_path(
&LineIndex::from_source_text(source_kind.source_code()) &LineIndex::from_source_text(source_kind.source_code())
), ),
&source_kind, &source_kind,
path,
) )
); );
} }

6
crates/ruff_dev/src/print_ast.rs
View file

@ -24,11 +24,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
args.file.display() args.file.display()
) )
})?; })?;
let python_ast = parse( let python_ast = parse(source_kind.source_code(), source_type.as_mode())?;
source_kind.source_code(),
source_type.as_mode(),
&args.file.to_string_lossy(),
)?;
println!("{python_ast:#?}"); println!("{python_ast:#?}");
Ok(()) Ok(())
} }

2
crates/ruff_dev/src/round_trip.rs
View file

@ -22,7 +22,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
println!("{}", ruff_notebook::round_trip(path)?); println!("{}", ruff_notebook::round_trip(path)?);
} else { } else {
let contents = fs::read_to_string(&args.file)?; let contents = fs::read_to_string(&args.file)?;
println!("{}", round_trip(&contents, &args.file.to_string_lossy())?); println!("{}", round_trip(&contents)?);
} }
Ok(()) Ok(())
} }

8
crates/ruff_linter/src/fix/edits.rs
View file

@ -397,13 +397,13 @@ mod tests {
#[test] #[test]
fn find_semicolon() -> Result<()> { fn find_semicolon() -> Result<()> {
let contents = "x = 1"; let contents = "x = 1";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!(trailing_semicolon(stmt.end(), &locator), None); assert_eq!(trailing_semicolon(stmt.end(), &locator), None);
let contents = "x = 1; y = 1"; let contents = "x = 1; y = 1";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(
@ -412,7 +412,7 @@ mod tests {
); );
let contents = "x = 1 ; y = 1"; let contents = "x = 1 ; y = 1";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(
@ -425,7 +425,7 @@ x = 1 \
; y = 1 ; y = 1
" "
.trim(); .trim();
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert_eq!( assert_eq!(

2
crates/ruff_linter/src/importer/insertion.rs
View file

@ -333,7 +333,7 @@ mod tests {
#[test] #[test]
fn start_of_file() -> Result<()> { fn start_of_file() -> Result<()> {
fn insert(contents: &str) -> Result<Insertion> { fn insert(contents: &str) -> Result<Insertion> {
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let tokens: Vec<LexResult> = ruff_python_parser::tokenize(contents, Mode::Module); let tokens: Vec<LexResult> = ruff_python_parser::tokenize(contents, Mode::Module);
let locator = Locator::new(contents); let locator = Locator::new(contents);
let stylist = Stylist::from_tokens(&tokens, &locator); let stylist = Stylist::from_tokens(&tokens, &locator);

6
crates/ruff_linter/src/linter.rs
View file

@ -145,7 +145,7 @@ pub fn check_path(
.any(|rule_code| rule_code.lint_source().is_imports()); .any(|rule_code| rule_code.lint_source().is_imports());
if use_ast || use_imports || use_doc_lines { if use_ast || use_imports || use_doc_lines {
// Parse, if the AST wasn't pre-provided provided. // Parse, if the AST wasn't pre-provided provided.
match tokens.into_ast_source(source_kind, source_type, path) { match tokens.into_ast_source(source_kind, source_type) {
Ok(python_ast) => { Ok(python_ast) => {
let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets); let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets);
if use_ast { if use_ast {
@ -339,7 +339,7 @@ pub fn add_noqa_to_path(
if let Some(error) = error { if let Some(error) = error {
error!( error!(
"{}", "{}",
DisplayParseError::new(error, locator.to_source_code(), source_kind) DisplayParseError::new(error, locator.to_source_code(), source_kind, path)
); );
} }
@ -687,13 +687,11 @@ impl<'a> TokenSource<'a> {
self, self,
source_kind: &SourceKind, source_kind: &SourceKind,
source_type: PySourceType, source_type: PySourceType,
path: &Path,
) -> Result<AstSource<'a>, ParseError> { ) -> Result<AstSource<'a>, ParseError> {
match self { match self {
Self::Tokens(tokens) => Ok(AstSource::Ast(ruff_python_parser::parse_program_tokens( Self::Tokens(tokens) => Ok(AstSource::Ast(ruff_python_parser::parse_program_tokens(
tokens, tokens,
source_kind.source_code(), source_kind.source_code(),
&path.to_string_lossy(),
source_type.is_ipynb(), source_type.is_ipynb(),
)?)), )?)),
Self::Precomputed { ast, .. } => Ok(AstSource::Precomputed(ast)), Self::Precomputed { ast, .. } => Ok(AstSource::Precomputed(ast)),

5
crates/ruff_linter/src/logging.rs
View file

@ -140,6 +140,7 @@ pub struct DisplayParseError<'a> {
error: ParseError, error: ParseError,
source_code: SourceCode<'a, 'a>, source_code: SourceCode<'a, 'a>,
source_kind: &'a SourceKind, source_kind: &'a SourceKind,
path: &'a Path,
} }
impl<'a> DisplayParseError<'a> { impl<'a> DisplayParseError<'a> {
@ -147,11 +148,13 @@ impl<'a> DisplayParseError<'a> {
error: ParseError, error: ParseError,
source_code: SourceCode<'a, 'a>, source_code: SourceCode<'a, 'a>,
source_kind: &'a SourceKind, source_kind: &'a SourceKind,
path: &'a Path,
) -> Self { ) -> Self {
Self { Self {
error, error,
source_code, source_code,
source_kind, source_kind,
path,
} }
} }
} }
@ -162,7 +165,7 @@ impl Display for DisplayParseError<'_> {
f, f,
"{header} {path}{colon}", "{header} {path}{colon}",
header = "Failed to parse".bold(), header = "Failed to parse".bold(),
path = fs::relativize_path(Path::new(&self.error.source_path)).bold(), path = fs::relativize_path(self.path).bold(),
colon = ":".cyan(), colon = ":".cyan(),
)?; )?;

2
crates/ruff_linter/src/rules/eradicate/detection.rs
View file

@ -74,7 +74,7 @@ pub(crate) fn comment_contains_code(line: &str, task_tags: &[String]) -> bool {
} }
// Finally, compile the source code. // Finally, compile the source code.
parse_suite(line, "<filename>").is_ok() parse_suite(line).is_ok()
} }
#[cfg(test)] #[cfg(test)]

2
crates/ruff_linter/src/rules/isort/rules/add_required_imports.rs
View file

@ -135,7 +135,7 @@ pub(crate) fn add_required_imports(
.required_imports .required_imports
.iter() .iter()
.flat_map(|required_import| { .flat_map(|required_import| {
let Ok(body) = parse_suite(required_import, "<filename>") else { let Ok(body) = parse_suite(required_import) else {
error!("Failed to parse required import: `{}`", required_import); error!("Failed to parse required import: `{}`", required_import);
return vec![]; return vec![];
}; };

34
crates/ruff_linter/src/rules/mccabe/rules/function_is_too_complex.rs
View file

@ -170,7 +170,7 @@ mod tests {
def trivial(): def trivial():
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -181,7 +181,7 @@ def trivial():
def expr_as_statement(): def expr_as_statement():
0xF00D 0xF00D
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -194,7 +194,7 @@ def sequential(n):
s = k + n s = k + n
return s return s
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -210,7 +210,7 @@ def if_elif_else_dead_path(n):
else: else:
return "smaller than or equal to three" return "smaller than or equal to three"
"#; "#;
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -227,7 +227,7 @@ def nested_ifs():
else: else:
return "smaller than or equal to three" return "smaller than or equal to three"
"#; "#;
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -239,7 +239,7 @@ def for_loop():
for i in range(10): for i in range(10):
print(i) print(i)
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -253,7 +253,7 @@ def for_else(mylist):
else: else:
print(None) print(None)
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -267,7 +267,7 @@ def recursive(n):
else: else:
return n return n
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -284,7 +284,7 @@ def nested_functions():
a() a()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -302,7 +302,7 @@ def try_else():
else: else:
print(4) print(4)
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 4); assert_eq!(get_complexity_number(&stmts), 4);
Ok(()) Ok(())
} }
@ -319,7 +319,7 @@ def nested_try_finally():
finally: finally:
print(3) print(3)
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -336,7 +336,7 @@ async def foobar(a, b, c):
async for x in a: async for x in a:
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 3); assert_eq!(get_complexity_number(&stmts), 3);
Ok(()) Ok(())
} }
@ -347,7 +347,7 @@ async def foobar(a, b, c):
def annotated_assign(): def annotated_assign():
x: Any = None x: Any = None
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -383,7 +383,7 @@ class Class:
return ServiceProvider(Logger()) return ServiceProvider(Logger())
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 9); assert_eq!(get_complexity_number(&stmts), 9);
Ok(()) Ok(())
} }
@ -397,7 +397,7 @@ def process_detect_lines():
finally: finally:
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 1); assert_eq!(get_complexity_number(&stmts), 1);
Ok(()) Ok(())
} }
@ -412,7 +412,7 @@ def process_detect_lines():
if res: if res:
errors.append(f"Non-zero exit code {res}") errors.append(f"Non-zero exit code {res}")
"#; "#;
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }
@ -425,7 +425,7 @@ def with_lock():
if foo: if foo:
print('bar') print('bar')
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(get_complexity_number(&stmts), 2); assert_eq!(get_complexity_number(&stmts), 2);
Ok(()) Ok(())
} }

2
crates/ruff_linter/src/rules/pylint/rules/too_many_branches.rs
View file

@ -180,7 +180,7 @@ mod tests {
use super::num_branches; use super::num_branches;
fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> { fn test_helper(source: &str, expected_num_branches: usize) -> Result<()> {
let branches = parse_suite(source, "<filename>")?; let branches = parse_suite(source)?;
assert_eq!(num_branches(&branches), expected_num_branches); assert_eq!(num_branches(&branches), expected_num_branches);
Ok(()) Ok(())
} }

2
crates/ruff_linter/src/rules/pylint/rules/too_many_return_statements.rs
View file

@ -103,7 +103,7 @@ mod tests {
use super::num_returns; use super::num_returns;
fn test_helper(source: &str, expected: usize) -> Result<()> { fn test_helper(source: &str, expected: usize) -> Result<()> {
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_returns(&stmts), expected); assert_eq!(num_returns(&stmts), expected);
Ok(()) Ok(())
} }

36
crates/ruff_linter/src/rules/pylint/rules/too_many_statements.rs
View file

@ -167,7 +167,7 @@ mod tests {
def f(): def f():
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }
@ -181,7 +181,7 @@ def f():
else: else:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -196,7 +196,7 @@ def f():
if a: if a:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 6); assert_eq!(num_statements(&stmts), 6);
Ok(()) Ok(())
} }
@ -210,7 +210,7 @@ def f():
elif a: elif a:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -228,7 +228,7 @@ def f():
else: else:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 9); assert_eq!(num_statements(&stmts), 9);
Ok(()) Ok(())
} }
@ -257,7 +257,7 @@ async def f():
import time import time
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 19); assert_eq!(num_statements(&stmts), 19);
Ok(()) Ok(())
} }
@ -269,7 +269,7 @@ def f():
for i in range(10): for i in range(10):
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }
@ -283,7 +283,7 @@ def f():
else: else:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 3); assert_eq!(num_statements(&stmts), 3);
Ok(()) Ok(())
} }
@ -298,7 +298,7 @@ def f():
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -316,7 +316,7 @@ def f():
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 3); assert_eq!(num_statements(&stmts), 3);
Ok(()) Ok(())
} }
@ -327,7 +327,7 @@ def f():
def f(): def f():
return return
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 1); assert_eq!(num_statements(&stmts), 1);
Ok(()) Ok(())
} }
@ -343,7 +343,7 @@ def f():
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 6); assert_eq!(num_statements(&stmts), 6);
Ok(()) Ok(())
} }
@ -357,7 +357,7 @@ def f():
except Exception: except Exception:
raise raise
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 5); assert_eq!(num_statements(&stmts), 5);
Ok(()) Ok(())
} }
@ -373,7 +373,7 @@ def f():
else: else:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 7); assert_eq!(num_statements(&stmts), 7);
Ok(()) Ok(())
} }
@ -391,7 +391,7 @@ def f():
finally: finally:
pass pass
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 10); assert_eq!(num_statements(&stmts), 10);
Ok(()) Ok(())
} }
@ -407,7 +407,7 @@ def f():
except Exception: except Exception:
raise raise
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 8); assert_eq!(num_statements(&stmts), 8);
Ok(()) Ok(())
} }
@ -425,7 +425,7 @@ def f():
finally: finally:
print() print()
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 11); assert_eq!(num_statements(&stmts), 11);
Ok(()) Ok(())
} }
@ -437,7 +437,7 @@ def f():
for i in range(10): for i in range(10):
yield i yield i
"; ";
let stmts = parse_suite(source, "<filename>")?; let stmts = parse_suite(source)?;
assert_eq!(num_statements(&stmts), 2); assert_eq!(num_statements(&stmts), 2);
Ok(()) Ok(())
} }

2
crates/ruff_linter/src/rules/ruff/rules/unreachable.rs
View file

@ -1053,7 +1053,7 @@ mod tests {
fn control_flow_graph(filename: &str) { fn control_flow_graph(filename: &str) {
let path = PathBuf::from_iter(["resources/test/fixtures/control-flow-graph", filename]); let path = PathBuf::from_iter(["resources/test/fixtures/control-flow-graph", filename]);
let source = fs::read_to_string(path).expect("failed to read file"); let source = fs::read_to_string(path).expect("failed to read file");
let stmts = parse(&source, Mode::Module, filename) let stmts = parse(&source, Mode::Module)
.unwrap_or_else(|err| panic!("failed to parse source: '{source}': {err}")) .unwrap_or_else(|err| panic!("failed to parse source: '{source}': {err}"))
.expect_module() .expect_module()
.body; .body;

2
crates/ruff_python_ast/tests/identifier.rs
View file

@ -12,7 +12,7 @@ else:
pass pass
" "
.trim(); .trim();
let stmts = parse_suite(contents, "<filename>")?; let stmts = parse_suite(contents)?;
let stmt = stmts.first().unwrap(); let stmt = stmts.first().unwrap();
let range = identifier::else_(stmt, contents).unwrap(); let range = identifier::else_(stmt, contents).unwrap();
assert_eq!(&contents[range], "else"); assert_eq!(&contents[range], "else");

16
crates/ruff_python_ast/tests/parenthesize.rs
View file

@ -6,7 +6,7 @@ use ruff_text_size::TextRange;
#[test] #[test]
fn test_parenthesized_name() { fn test_parenthesized_name() {
let source_code = r"(x) + 1"; let source_code = r"(x) + 1";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap(); let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref(); let name = bin_op.left.as_ref();
@ -23,7 +23,7 @@ fn test_parenthesized_name() {
#[test] #[test]
fn test_non_parenthesized_name() { fn test_non_parenthesized_name() {
let source_code = r"x + 1"; let source_code = r"x + 1";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap(); let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref(); let name = bin_op.left.as_ref();
@ -40,7 +40,7 @@ fn test_non_parenthesized_name() {
#[test] #[test]
fn test_parenthesized_argument() { fn test_parenthesized_argument() {
let source_code = r"f((a))"; let source_code = r"f((a))";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap(); let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments; let arguments = &call.arguments;
@ -58,7 +58,7 @@ fn test_parenthesized_argument() {
#[test] #[test]
fn test_non_parenthesized_argument() { fn test_non_parenthesized_argument() {
let source_code = r"f(a)"; let source_code = r"f(a)";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap(); let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments; let arguments = &call.arguments;
@ -76,7 +76,7 @@ fn test_non_parenthesized_argument() {
#[test] #[test]
fn test_parenthesized_tuple_member() { fn test_parenthesized_tuple_member() {
let source_code = r"(a, (b))"; let source_code = r"(a, (b))";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let tuple = expr.as_tuple_expr().unwrap(); let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap(); let member = tuple.elts.last().unwrap();
@ -93,7 +93,7 @@ fn test_parenthesized_tuple_member() {
#[test] #[test]
fn test_non_parenthesized_tuple_member() { fn test_non_parenthesized_tuple_member() {
let source_code = r"(a, b)"; let source_code = r"(a, b)";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let tuple = expr.as_tuple_expr().unwrap(); let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap(); let member = tuple.elts.last().unwrap();
@ -110,7 +110,7 @@ fn test_non_parenthesized_tuple_member() {
#[test] #[test]
fn test_twice_parenthesized_name() { fn test_twice_parenthesized_name() {
let source_code = r"((x)) + 1"; let source_code = r"((x)) + 1";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let bin_op = expr.as_bin_op_expr().unwrap(); let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref(); let name = bin_op.left.as_ref();
@ -127,7 +127,7 @@ fn test_twice_parenthesized_name() {
#[test] #[test]
fn test_twice_parenthesized_argument() { fn test_twice_parenthesized_argument() {
let source_code = r"f(((a + 1)))"; let source_code = r"f(((a + 1)))";
let expr = parse_expression(source_code, "<filename>").unwrap(); let expr = parse_expression(source_code).unwrap();
let call = expr.as_call_expr().unwrap(); let call = expr.as_call_expr().unwrap();
let arguments = &call.arguments; let arguments = &call.arguments;

2
crates/ruff_python_ast/tests/preorder.rs
View file

@ -149,7 +149,7 @@ fn f_strings() {
fn trace_preorder_visitation(source: &str) -> String { fn trace_preorder_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module); let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, source, Mode::Module, "test.py").unwrap(); let parsed = parse_tokens(tokens, source, Mode::Module).unwrap();
let mut visitor = RecordVisitor::default(); let mut visitor = RecordVisitor::default();
visitor.visit_mod(&parsed); visitor.visit_mod(&parsed);

4
crates/ruff_python_ast/tests/stmt_if.rs
View file

@ -10,7 +10,7 @@ fn extract_elif_else_range() -> Result<(), ParseError> {
elif b: elif b:
... ...
"; ";
let mut stmts = parse_suite(contents, "<filename>")?; let mut stmts = parse_suite(contents)?;
let stmt = stmts let stmt = stmts
.pop() .pop()
.and_then(ruff_python_ast::Stmt::if_stmt) .and_then(ruff_python_ast::Stmt::if_stmt)
@ -24,7 +24,7 @@ elif b:
else: else:
... ...
"; ";
let mut stmts = parse_suite(contents, "<filename>")?; let mut stmts = parse_suite(contents)?;
let stmt = stmts let stmt = stmts
.pop() .pop()
.and_then(ruff_python_ast::Stmt::if_stmt) .and_then(ruff_python_ast::Stmt::if_stmt)

2
crates/ruff_python_ast/tests/visitor.rs
View file

@ -160,7 +160,7 @@ fn f_strings() {
fn trace_visitation(source: &str) -> String { fn trace_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module); let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, source, Mode::Module, "test.py").unwrap(); let parsed = parse_tokens(tokens, source, Mode::Module).unwrap();
let mut visitor = RecordVisitor::default(); let mut visitor = RecordVisitor::default();
walk_module(&mut visitor, &parsed); walk_module(&mut visitor, &parsed);

6
crates/ruff_python_codegen/src/generator.rs
View file

@ -1416,7 +1416,7 @@ mod tests {
let indentation = Indentation::default(); let indentation = Indentation::default();
let quote = Quote::default(); let quote = Quote::default();
let line_ending = LineEnding::default(); let line_ending = LineEnding::default();
let stmt = parse_suite(contents, "<filename>").unwrap(); let stmt = parse_suite(contents).unwrap();
let mut generator = Generator::new(&indentation, quote, line_ending); let mut generator = Generator::new(&indentation, quote, line_ending);
generator.unparse_suite(&stmt); generator.unparse_suite(&stmt);
generator.generate() generator.generate()
@ -1428,7 +1428,7 @@ mod tests {
line_ending: LineEnding, line_ending: LineEnding,
contents: &str, contents: &str,
) -> String { ) -> String {
let stmt = parse_suite(contents, "<filename>").unwrap(); let stmt = parse_suite(contents).unwrap();
let mut generator = Generator::new(indentation, quote, line_ending); let mut generator = Generator::new(indentation, quote, line_ending);
generator.unparse_suite(&stmt); generator.unparse_suite(&stmt);
generator.generate() generator.generate()
@ -1438,7 +1438,7 @@ mod tests {
let indentation = Indentation::default(); let indentation = Indentation::default();
let quote = Quote::default(); let quote = Quote::default();
let line_ending = LineEnding::default(); let line_ending = LineEnding::default();
let ast = ruff_python_parser::parse(contents, Mode::Ipython, "<filename>").unwrap(); let ast = ruff_python_parser::parse(contents, Mode::Ipython).unwrap();
let Mod::Module(ModModule { body, .. }) = ast else { let Mod::Module(ModModule { body, .. }) = ast else {
panic!("Source code didn't return ModModule") panic!("Source code didn't return ModModule")
}; };

4
crates/ruff_python_codegen/src/lib.rs
View file

@ -7,9 +7,9 @@ use ruff_source_file::Locator;
pub use stylist::{Quote, Stylist}; pub use stylist::{Quote, Stylist};
/// Run round-trip source code generation on a given Python code. /// Run round-trip source code generation on a given Python code.
pub fn round_trip(code: &str, source_path: &str) -> Result<String, ParseError> { pub fn round_trip(code: &str) -> Result<String, ParseError> {
let locator = Locator::new(code); let locator = Locator::new(code);
let python_ast = parse_suite(code, source_path)?; let python_ast = parse_suite(code)?;
let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect(); let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect();
let stylist = Stylist::from_tokens(&tokens, &locator); let stylist = Stylist::from_tokens(&tokens, &locator);
let mut generator: Generator = (&stylist).into(); let mut generator: Generator = (&stylist).into();

4
crates/ruff_python_formatter/src/cli.rs
View file

@ -50,8 +50,8 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
.map_err(|err| format_err!("Source contains syntax errors {err:?}"))?; .map_err(|err| format_err!("Source contains syntax errors {err:?}"))?;
// Parse the AST. // Parse the AST.
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>") let module =
.context("Syntax error in input")?; parse_ok_tokens(tokens, source, source_type.as_mode()).context("Syntax error in input")?;
let options = PyFormatOptions::from_extension(source_path) let options = PyFormatOptions::from_extension(source_path)
.with_preview(if cli.preview { .with_preview(if cli.preview {

2
crates/ruff_python_formatter/src/comments/mod.rs
View file

@ -568,7 +568,7 @@ mod tests {
let source_type = PySourceType::Python; let source_type = PySourceType::Python;
let (tokens, comment_ranges) = let (tokens, comment_ranges) =
tokens_and_ranges(source, source_type).expect("Expect source to be valid Python"); tokens_and_ranges(source, source_type).expect("Expect source to be valid Python");
let parsed = parse_ok_tokens(tokens, source, source_type.as_mode(), "test.py") let parsed = parse_ok_tokens(tokens, source, source_type.as_mode())
.expect("Expect source to be valid Python"); .expect("Expect source to be valid Python");
CommentsTestCase { CommentsTestCase {

2
crates/ruff_python_formatter/src/expression/expr_name.rs
View file

@ -52,7 +52,7 @@ mod tests {
#[test] #[test]
fn name_range_with_comments() { fn name_range_with_comments() {
let source = parse_program("a # comment", "file.py").unwrap(); let source = parse_program("a # comment").unwrap();
let expression_statement = source let expression_statement = source
.body .body

2
crates/ruff_python_formatter/src/expression/parentheses.rs
View file

@ -447,7 +447,7 @@ mod tests {
#[test] #[test]
fn test_has_parentheses() { fn test_has_parentheses() {
let expression = r#"(b().c("")).d()"#; let expression = r#"(b().c("")).d()"#;
let expr = parse_expression(expression, "<filename>").unwrap(); let expr = parse_expression(expression).unwrap();
assert!(!is_expression_parenthesized( assert!(!is_expression_parenthesized(
ExpressionRef::from(&expr), ExpressionRef::from(&expr),
&CommentRanges::default(), &CommentRanges::default(),

4
crates/ruff_python_formatter/src/lib.rs
View file

@ -137,7 +137,7 @@ pub fn format_module_source(
) -> Result<Printed, FormatModuleError> { ) -> Result<Printed, FormatModuleError> {
let source_type = options.source_type(); let source_type = options.source_type();
let (tokens, comment_ranges) = tokens_and_ranges(source, source_type)?; let (tokens, comment_ranges) = tokens_and_ranges(source, source_type)?;
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>")?; let module = parse_ok_tokens(tokens, source, source_type.as_mode())?;
let formatted = format_module_ast(&module, &comment_ranges, source, options)?; let formatted = format_module_ast(&module, &comment_ranges, source, options)?;
Ok(formatted.print()?) Ok(formatted.print()?)
} }
@ -225,7 +225,7 @@ def main() -> None:
// Parse the AST. // Parse the AST.
let source_path = "code_inline.py"; let source_path = "code_inline.py";
let module = parse_ok_tokens(tokens, source, source_type.as_mode(), source_path).unwrap(); let module = parse_ok_tokens(tokens, source, source_type.as_mode()).unwrap();
let options = PyFormatOptions::from_extension(Path::new(source_path)); let options = PyFormatOptions::from_extension(Path::new(source_path));
let formatted = format_module_ast(&module, &comment_ranges, source, options).unwrap(); let formatted = format_module_ast(&module, &comment_ranges, source, options).unwrap();

2
crates/ruff_python_formatter/src/statement/suite.rs
View file

@ -746,7 +746,7 @@ def trailing_func():
pass pass
"; ";
let statements = parse_suite(source, "test.py").unwrap(); let statements = parse_suite(source).unwrap();
let comment_ranges = CommentRanges::default(); let comment_ranges = CommentRanges::default();
let context = PyFormatContext::new( let context = PyFormatContext::new(

9
crates/ruff_python_formatter/src/string/docstring.rs
View file

@ -524,11 +524,7 @@ impl<'ast, 'buf, 'fmt, 'src> DocstringLinePrinter<'ast, 'buf, 'fmt, 'src> {
std::format!(r#""""{}""""#, printed.as_code()) std::format!(r#""""{}""""#, printed.as_code())
} }
}; };
let result = ruff_python_parser::parse( let result = ruff_python_parser::parse(&wrapped, self.f.options().source_type().as_mode());
&wrapped,
self.f.options().source_type().as_mode(),
"<filename>",
);
// If the resulting code is not valid, then reset and pass through // If the resulting code is not valid, then reset and pass through
// the docstring lines as-is. // the docstring lines as-is.
if result.is_err() { if result.is_err() {
@ -1523,8 +1519,7 @@ fn docstring_format_source(
let source_type = options.source_type(); let source_type = options.source_type();
let (tokens, comment_ranges) = ruff_python_index::tokens_and_ranges(source, source_type)?; let (tokens, comment_ranges) = ruff_python_index::tokens_and_ranges(source, source_type)?;
let module = let module = ruff_python_parser::parse_ok_tokens(tokens, source, source_type.as_mode())?;
ruff_python_parser::parse_ok_tokens(tokens, source, source_type.as_mode(), "<filename>")?;
let source_code = ruff_formatter::SourceCode::new(source); let source_code = ruff_formatter::SourceCode::new(source);
let comments = crate::Comments::from_ast(&module, source_code, &comment_ranges); let comments = crate::Comments::from_ast(&module, source_code, &comment_ranges);
let locator = Locator::new(source); let locator = Locator::new(source);

14
crates/ruff_python_formatter/tests/fixtures.rs
View file

@ -300,22 +300,14 @@ fn ensure_unchanged_ast(
let source_type = options.source_type(); let source_type = options.source_type();
// Parse the unformatted code. // Parse the unformatted code.
let mut unformatted_ast = parse( let mut unformatted_ast = parse(unformatted_code, source_type.as_mode())
unformatted_code,
source_type.as_mode(),
&input_path.to_string_lossy(),
)
.expect("Unformatted code to be valid syntax"); .expect("Unformatted code to be valid syntax");
Normalizer.visit_module(&mut unformatted_ast); Normalizer.visit_module(&mut unformatted_ast);
let unformatted_ast = ComparableMod::from(&unformatted_ast); let unformatted_ast = ComparableMod::from(&unformatted_ast);
// Parse the formatted code. // Parse the formatted code.
let mut formatted_ast = parse( let mut formatted_ast =
formatted_code, parse(formatted_code, source_type.as_mode()).expect("Formatted code to be valid syntax");
source_type.as_mode(),
&input_path.to_string_lossy(),
)
.expect("Formatted code to be valid syntax");
Normalizer.visit_module(&mut formatted_ast); Normalizer.visit_module(&mut formatted_ast);
let formatted_ast = ComparableMod::from(&formatted_ast); let formatted_ast = ComparableMod::from(&formatted_ast);

36
crates/ruff_python_parser/src/context.rs
View file

@ -54,126 +54,126 @@ mod tests {
#[test] #[test]
fn test_assign_name() { fn test_assign_name() {
let source = "x = (1, 2, 3)"; let source = "x = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_tuple() { fn test_assign_tuple() {
let source = "(x, y) = (1, 2, 3)"; let source = "(x, y) = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_list() { fn test_assign_list() {
let source = "[x, y] = (1, 2, 3)"; let source = "[x, y] = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_attribute() { fn test_assign_attribute() {
let source = "x.y = (1, 2, 3)"; let source = "x.y = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_subscript() { fn test_assign_subscript() {
let source = "x[y] = (1, 2, 3)"; let source = "x[y] = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_starred() { fn test_assign_starred() {
let source = "(x, *y) = (1, 2, 3)"; let source = "(x, *y) = (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_for() { fn test_assign_for() {
let source = "for x in (1, 2, 3): pass"; let source = "for x in (1, 2, 3): pass";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_list_comp() { fn test_assign_list_comp() {
let source = "x = [y for y in (1, 2, 3)]"; let source = "x = [y for y in (1, 2, 3)]";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_set_comp() { fn test_assign_set_comp() {
let source = "x = {y for y in (1, 2, 3)}"; let source = "x = {y for y in (1, 2, 3)}";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_with() { fn test_assign_with() {
let source = "with 1 as x: pass"; let source = "with 1 as x: pass";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_assign_named_expr() { fn test_assign_named_expr() {
let source = "if x:= 1: pass"; let source = "if x:= 1: pass";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_ann_assign_name() { fn test_ann_assign_name() {
let source = "x: int = 1"; let source = "x: int = 1";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_name() { fn test_aug_assign_name() {
let source = "x += 1"; let source = "x += 1";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_attribute() { fn test_aug_assign_attribute() {
let source = "x.y += (1, 2, 3)"; let source = "x.y += (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_aug_assign_subscript() { fn test_aug_assign_subscript() {
let source = "x[y] += (1, 2, 3)"; let source = "x[y] += (1, 2, 3)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_name() { fn test_del_name() {
let source = "del x"; let source = "del x";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_attribute() { fn test_del_attribute() {
let source = "del x.y"; let source = "del x.y";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_del_subscript() { fn test_del_subscript() {
let source = "del x[y]"; let source = "del x[y]";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
} }

4
crates/ruff_python_parser/src/function.rs
View file

@ -148,7 +148,7 @@ mod tests {
$( $(
#[test] #[test]
fn $name() { fn $name() {
let parse_ast = crate::parser::parse_suite($code, "<test>"); let parse_ast = crate::parser::parse_suite($code, );
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
)* )*
@ -179,7 +179,7 @@ mod tests {
} }
fn function_parse_error(src: &str) -> LexicalErrorType { fn function_parse_error(src: &str) -> LexicalErrorType {
let parse_ast = parse_suite(src, "<test>"); let parse_ast = parse_suite(src);
parse_ast parse_ast
.map_err(|e| match e.error { .map_err(|e| match e.error {
ParseErrorType::Lexical(e) => e, ParseErrorType::Lexical(e) => e,

255
crates/ruff_python_parser/src/invalid.rs
View file

@ -100,26 +100,25 @@ mod tests {
// Regression test: https://github.com/astral-sh/ruff/issues/6895 // Regression test: https://github.com/astral-sh/ruff/issues/6895
#[test] #[test]
fn err_literal_assignment() { fn err_literal_assignment() {
let ast = parse_suite(r"5 = 3", "<test>"); let ast = parse_suite(r"5 = 3");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
// This test previously passed before the assignment operator checking // This test previously passed before the assignment operator checking
// above, but we include it here for good measure. // above, but we include it here for good measure.
#[test] #[test]
fn err_assignment_expr() { fn err_assignment_expr() {
let ast = parse_suite(r"(5 := 3)", "<test>"); let ast = parse_suite(r"(5 := 3)");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: UnrecognizedToken( error: UnrecognizedToken(
@ -127,316 +126,296 @@ Err(
None, None,
), ),
offset: 3, offset: 3,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_literal_augment_assignment() { fn err_literal_augment_assignment() {
let ast = parse_suite(r"5 += 3", "<test>"); let ast = parse_suite(r"5 += 3");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_literal_annotation_assignment() { fn err_literal_annotation_assignment() {
let ast = parse_suite(r"(5): int = 3", "<test>"); let ast = parse_suite(r"(5): int = 3");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
// Now we exhaustively test all possible cases where assignment can fail. // Now we exhaustively test all possible cases where assignment can fail.
#[test] #[test]
fn err_bool_op() { fn err_bool_op() {
let ast = parse_suite(r"x or y = 42", "<test>"); let ast = parse_suite(r"x or y = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_named_expr() { fn err_named_expr() {
let ast = parse_suite(r"(x := 5) = 42", "<test>"); let ast = parse_suite(r"(x := 5) = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_bin_op() { fn err_bin_op() {
let ast = parse_suite(r"x + y = 42", "<test>"); let ast = parse_suite(r"x + y = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_unary_op() { fn err_unary_op() {
let ast = parse_suite(r"-x = 42", "<test>"); let ast = parse_suite(r"-x = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_lambda() { fn err_lambda() {
let ast = parse_suite(r"(lambda _: 1) = 42", "<test>"); let ast = parse_suite(r"(lambda _: 1) = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_if_exp() { fn err_if_exp() {
let ast = parse_suite(r"a if b else c = 42", "<test>"); let ast = parse_suite(r"a if b else c = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_dict() { fn err_dict() {
let ast = parse_suite(r"{'a':5} = 42", "<test>"); let ast = parse_suite(r"{'a':5} = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_set() { fn err_set() {
let ast = parse_suite(r"{a} = 42", "<test>"); let ast = parse_suite(r"{a} = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_list_comp() { fn err_list_comp() {
let ast = parse_suite(r"[x for x in xs] = 42", "<test>"); let ast = parse_suite(r"[x for x in xs] = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_set_comp() { fn err_set_comp() {
let ast = parse_suite(r"{x for x in xs} = 42", "<test>"); let ast = parse_suite(r"{x for x in xs} = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_dict_comp() { fn err_dict_comp() {
let ast = parse_suite(r"{x: x*2 for x in xs} = 42", "<test>"); let ast = parse_suite(r"{x: x*2 for x in xs} = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_generator_exp() { fn err_generator_exp() {
let ast = parse_suite(r"(x for x in xs) = 42", "<test>"); let ast = parse_suite(r"(x for x in xs) = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_await() { fn err_await() {
let ast = parse_suite(r"await x = 42", "<test>"); let ast = parse_suite(r"await x = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_yield() { fn err_yield() {
let ast = parse_suite(r"(yield x) = 42", "<test>"); let ast = parse_suite(r"(yield x) = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_yield_from() { fn err_yield_from() {
let ast = parse_suite(r"(yield from xs) = 42", "<test>"); let ast = parse_suite(r"(yield from xs) = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_compare() { fn err_compare() {
let ast = parse_suite(r"a < b < c = 42", "<test>"); let ast = parse_suite(r"a < b < c = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_call() { fn err_call() {
let ast = parse_suite(r"foo() = 42", "<test>"); let ast = parse_suite(r"foo() = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
@ -452,194 +431,182 @@ Err(
// is coupled. // is coupled.
// //
// See: https://docs.python.org/3/library/ast.html#ast.FormattedValue // See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
let ast = parse_suite(r#"f"{quux}" = 42"#, "<test>"); let ast = parse_suite(r#"f"{quux}" = 42"#);
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_fstring() { fn err_fstring() {
let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#, "<test>"); let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#);
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_string_literal() { fn err_string_literal() {
let ast = parse_suite(r#""foo" = 42"#, "<test>"); let ast = parse_suite(r#""foo" = 42"#);
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_bytes_literal() { fn err_bytes_literal() {
let ast = parse_suite(r#"b"foo" = 42"#, "<test>"); let ast = parse_suite(r#"b"foo" = 42"#);
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_number_literal() { fn err_number_literal() {
let ast = parse_suite(r"123 = 42", "<test>"); let ast = parse_suite(r"123 = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_boolean_literal() { fn err_boolean_literal() {
let ast = parse_suite(r"True = 42", "<test>"); let ast = parse_suite(r"True = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_none_literal() { fn err_none_literal() {
let ast = parse_suite(r"None = 42", "<test>"); let ast = parse_suite(r"None = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_ellipsis_literal() { fn err_ellipsis_literal() {
let ast = parse_suite(r"... = 42", "<test>"); let ast = parse_suite(r"... = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 0, offset: 0,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_starred() { fn err_starred() {
let ast = parse_suite(r"*foo() = 42", "<test>"); let ast = parse_suite(r"*foo() = 42");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 1, offset: 1,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_list() { fn err_list() {
let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]", "<test>"); let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 4, offset: 4,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_list_nested() { fn err_list_nested() {
let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]", "<test>"); let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 11, offset: 11,
source_path: "<test>",
}, },
) )
"#); "###);
} }
#[test] #[test]
fn err_tuple() { fn err_tuple() {
let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)", "<test>"); let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)");
insta::assert_debug_snapshot!(ast, @r#" insta::assert_debug_snapshot!(ast, @r###"
Err( Err(
ParseError { ParseError {
error: Lexical( error: Lexical(
AssignmentError, AssignmentError,
), ),
offset: 4, offset: 4,
source_path: "<test>",
}, },
) )
"#); "###);
} }
// This last group of tests checks that assignments we expect to be parsed // This last group of tests checks that assignments we expect to be parsed
@ -647,25 +614,25 @@ Err(
#[test] #[test]
fn ok_starred() { fn ok_starred() {
let ast = parse_suite(r"*foo = 42", "<test>"); let ast = parse_suite(r"*foo = 42");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_list() { fn ok_list() {
let ast = parse_suite(r"[x, y, z] = [1, 2, 3]", "<test>"); let ast = parse_suite(r"[x, y, z] = [1, 2, 3]");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_tuple() { fn ok_tuple() {
let ast = parse_suite(r"(x, y, z) = (1, 2, 3)", "<test>"); let ast = parse_suite(r"(x, y, z) = (1, 2, 3)");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_subscript_normal() { fn ok_subscript_normal() {
let ast = parse_suite(r"x[0] = 42", "<test>"); let ast = parse_suite(r"x[0] = 42");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
@ -673,13 +640,13 @@ Err(
// doesn't fail parsing. // doesn't fail parsing.
#[test] #[test]
fn ok_subscript_weird() { fn ok_subscript_weird() {
let ast = parse_suite(r"5[0] = 42", "<test>"); let ast = parse_suite(r"5[0] = 42");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_slice_normal() { fn ok_slice_normal() {
let ast = parse_suite(r"x[1:2] = [42]", "<test>"); let ast = parse_suite(r"x[1:2] = [42]");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
@ -687,13 +654,13 @@ Err(
// doesn't fail parsing. // doesn't fail parsing.
#[test] #[test]
fn ok_slice_weird() { fn ok_slice_weird() {
let ast = parse_suite(r"5[1:2] = [42]", "<test>"); let ast = parse_suite(r"5[1:2] = [42]");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_attribute_normal() { fn ok_attribute_normal() {
let ast = parse_suite(r"foo.bar = 42", "<test>"); let ast = parse_suite(r"foo.bar = 42");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
@ -701,13 +668,13 @@ Err(
// it doesn't fail parsing. // it doesn't fail parsing.
#[test] #[test]
fn ok_attribute_weird() { fn ok_attribute_weird() {
let ast = parse_suite(r#""foo".y = 42"#, "<test>"); let ast = parse_suite(r#""foo".y = 42"#);
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_name() { fn ok_name() {
let ast = parse_suite(r"foo = 42", "<test>"); let ast = parse_suite(r"foo = 42");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
@ -720,13 +687,13 @@ Err(
let src = r"!foo = 42"; let src = r"!foo = 42";
let tokens = crate::lexer::lex(src, Mode::Ipython); let tokens = crate::lexer::lex(src, Mode::Ipython);
let ast = crate::parse_tokens(tokens, src, Mode::Ipython, "<test>"); let ast = crate::parse_tokens(tokens, src, Mode::Ipython);
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
#[test] #[test]
fn ok_assignment_expr() { fn ok_assignment_expr() {
let ast = parse_suite(r"(x := 5)", "<test>"); let ast = parse_suite(r"(x := 5)");
insta::assert_debug_snapshot!(ast); insta::assert_debug_snapshot!(ast);
} }
} }

21
crates/ruff_python_parser/src/lib.rs
View file

@ -85,7 +85,7 @@
//! return bool(i & 1) //! return bool(i & 1)
//! "#; //! "#;
//! let tokens = lex(python_source, Mode::Module); //! let tokens = lex(python_source, Mode::Module);
//! let ast = parse_tokens(tokens, python_source, Mode::Module, "<embedded>"); //! let ast = parse_tokens(tokens, python_source, Mode::Module);
//! //!
//! assert!(ast.is_ok()); //! assert!(ast.is_ok());
//! ``` //! ```
@ -100,7 +100,7 @@
//! def is_odd(i): //! def is_odd(i):
//! return bool(i & 1) //! return bool(i & 1)
//! "#; //! "#;
//! let ast = parse_suite(python_source, "<embedded>"); //! let ast = parse_suite(python_source);
//! //!
//! assert!(ast.is_ok()); //! assert!(ast.is_ok());
//! ``` //! ```
@ -148,7 +148,6 @@ pub fn tokenize(contents: &str, mode: Mode) -> Vec<LexResult> {
pub fn parse_program_tokens( pub fn parse_program_tokens(
lxr: Vec<LexResult>, lxr: Vec<LexResult>,
source: &str, source: &str,
source_path: &str,
is_jupyter_notebook: bool, is_jupyter_notebook: bool,
) -> anyhow::Result<Suite, ParseError> { ) -> anyhow::Result<Suite, ParseError> {
let mode = if is_jupyter_notebook { let mode = if is_jupyter_notebook {
@ -156,7 +155,7 @@ pub fn parse_program_tokens(
} else { } else {
Mode::Module Mode::Module
}; };
match parse_tokens(lxr, source, mode, source_path)? { match parse_tokens(lxr, source, mode)? {
Mod::Module(m) => Ok(m.body), Mod::Module(m) => Ok(m.body),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"), Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
} }
@ -379,7 +378,7 @@ mod tests {
#[test] #[test]
fn extract_cmp_op_location() -> Result<()> { fn extract_cmp_op_location() -> Result<()> {
let contents = "x == 1"; let contents = "x == 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -389,7 +388,7 @@ mod tests {
); );
let contents = "x != 1"; let contents = "x != 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -399,7 +398,7 @@ mod tests {
); );
let contents = "x is 1"; let contents = "x is 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -409,7 +408,7 @@ mod tests {
); );
let contents = "x is not 1"; let contents = "x is not 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -419,7 +418,7 @@ mod tests {
); );
let contents = "x in 1"; let contents = "x in 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -429,7 +428,7 @@ mod tests {
); );
let contents = "x not in 1"; let contents = "x not in 1";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(
@ -439,7 +438,7 @@ mod tests {
); );
let contents = "x != (1 is not 2)"; let contents = "x != (1 is not 2)";
let expr = parse_expression(contents, "<filename>")?; let expr = parse_expression(contents)?;
assert_eq!( assert_eq!(
locate_cmp_ops(&expr, contents), locate_cmp_ops(&expr, contents),
vec![LocatedCmpOp::new( vec![LocatedCmpOp::new(

170
crates/ruff_python_parser/src/parser.rs
View file

@ -12,8 +12,6 @@
//! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree //! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
//! [`Mode`]: crate::mode //! [`Mode`]: crate::mode
use std::{fmt, iter};
use itertools::Itertools; use itertools::Itertools;
pub(super) use lalrpop_util::ParseError as LalrpopError; pub(super) use lalrpop_util::ParseError as LalrpopError;
use ruff_text_size::{Ranged, TextRange, TextSize}; use ruff_text_size::{Ranged, TextRange, TextSize};
@ -51,19 +49,19 @@ use ruff_python_ast::{
/// ///
/// print(foo()) /// print(foo())
/// "#; /// "#;
/// let program = parser::parse_program(source, "<embedded>"); /// let program = parser::parse_program(source);
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
pub fn parse_program(source: &str, source_path: &str) -> Result<ModModule, ParseError> { pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
let lexer = lex(source, Mode::Module); let lexer = lex(source, Mode::Module);
match parse_tokens(lexer, source, Mode::Module, source_path)? { match parse_tokens(lexer, source, Mode::Module)? {
Mod::Module(m) => Ok(m), Mod::Module(m) => Ok(m),
Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"), Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
} }
} }
pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError> { pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
parse_program(source, source_path).map(|m| m.body) parse_program(source).map(|m| m.body)
} }
/// Parses a single Python expression. /// Parses a single Python expression.
@ -77,14 +75,14 @@ pub fn parse_suite(source: &str, source_path: &str) -> Result<Suite, ParseError>
/// ///
/// ``` /// ```
/// use ruff_python_parser as parser; /// use ruff_python_parser as parser;
/// let expr = parser::parse_expression("1 + 2", "<embedded>"); /// let expr = parser::parse_expression("1 + 2");
/// ///
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ///
/// ``` /// ```
pub fn parse_expression(source: &str, source_path: &str) -> Result<Expr, ParseError> { pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
let lexer = lex(source, Mode::Expression); let lexer = lex(source, Mode::Expression);
match parse_tokens(lexer, source, Mode::Expression, source_path)? { match parse_tokens(lexer, source, Mode::Expression)? {
Mod::Expression(expression) => Ok(*expression.body), Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"), Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
} }
@ -104,16 +102,12 @@ pub fn parse_expression(source: &str, source_path: &str) -> Result<Expr, ParseEr
/// use ruff_python_parser::{parse_expression_starts_at}; /// use ruff_python_parser::{parse_expression_starts_at};
/// # use ruff_text_size::TextSize; /// # use ruff_text_size::TextSize;
/// ///
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400)); /// let expr = parse_expression_starts_at("1 + 2", TextSize::from(400));
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ``` /// ```
pub fn parse_expression_starts_at( pub fn parse_expression_starts_at(source: &str, offset: TextSize) -> Result<Expr, ParseError> {
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Expr, ParseError> {
let lexer = lex_starts_at(source, Mode::Module, offset); let lexer = lex_starts_at(source, Mode::Module, offset);
match parse_tokens(lexer, source, Mode::Expression, source_path)? { match parse_tokens(lexer, source, Mode::Expression)? {
Mod::Expression(expression) => Ok(*expression.body), Mod::Expression(expression) => Ok(*expression.body),
Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"), Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
} }
@ -133,7 +127,7 @@ pub fn parse_expression_starts_at(
/// ``` /// ```
/// use ruff_python_parser::{Mode, parse}; /// use ruff_python_parser::{Mode, parse};
/// ///
/// let expr = parse("1 + 2", Mode::Expression, "<embedded>"); /// let expr = parse("1 + 2", Mode::Expression);
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ``` /// ```
/// ///
@ -148,7 +142,7 @@ pub fn parse_expression_starts_at(
/// def greet(self): /// def greet(self):
/// print("Hello, world!") /// print("Hello, world!")
/// "#; /// "#;
/// let program = parse(source, Mode::Module, "<embedded>"); /// let program = parse(source, Mode::Module);
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
/// ///
@ -162,11 +156,11 @@ pub fn parse_expression_starts_at(
/// ?str.replace /// ?str.replace
/// !ls /// !ls
/// "#; /// "#;
/// let program = parse(source, Mode::Ipython, "<embedded>"); /// let program = parse(source, Mode::Ipython);
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<Mod, ParseError> { pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
parse_starts_at(source, mode, source_path, TextSize::default()) parse_starts_at(source, mode, TextSize::default())
} }
/// Parse the given Python source code using the specified [`Mode`] and [`TextSize`]. /// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
@ -189,17 +183,12 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<Mod, ParseEr
/// ///
/// print(fib(42)) /// print(fib(42))
/// "#; /// "#;
/// let program = parse_starts_at(source, Mode::Module, "<embedded>", TextSize::from(0)); /// let program = parse_starts_at(source, Mode::Module, TextSize::from(0));
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
pub fn parse_starts_at( pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod, ParseError> {
source: &str,
mode: Mode,
source_path: &str,
offset: TextSize,
) -> Result<Mod, ParseError> {
let lxr = lexer::lex_starts_at(source, mode, offset); let lxr = lexer::lex_starts_at(source, mode, offset);
parse_tokens(lxr, source, mode, source_path) parse_tokens(lxr, source, mode)
} }
/// Parse an iterator of [`LexResult`]s using the specified [`Mode`]. /// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
@ -215,14 +204,13 @@ pub fn parse_starts_at(
/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens}; /// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
/// ///
/// let source = "1 + 2"; /// let source = "1 + 2";
/// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression, "<embedded>"); /// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression);
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ``` /// ```
pub fn parse_tokens( pub fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>, lxr: impl IntoIterator<Item = LexResult>,
source: &str, source: &str,
mode: Mode, mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> { ) -> Result<Mod, ParseError> {
let lxr = lxr.into_iter(); let lxr = lxr.into_iter();
@ -230,7 +218,6 @@ pub fn parse_tokens(
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)), lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
source, source,
mode, mode,
source_path,
) )
} }
@ -239,35 +226,33 @@ pub fn parse_ok_tokens(
lxr: impl IntoIterator<Item = Spanned>, lxr: impl IntoIterator<Item = Spanned>,
source: &str, source: &str,
mode: Mode, mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> { ) -> Result<Mod, ParseError> {
let lxr = lxr let lxr = lxr
.into_iter() .into_iter()
.filter(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); .filter(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
let marker_token = (Tok::start_marker(mode), TextRange::default()); let marker_token = (Tok::start_marker(mode), TextRange::default());
let lexer = iter::once(marker_token) let lexer = std::iter::once(marker_token)
.chain(lxr) .chain(lxr)
.map(|(t, range)| (range.start(), t, range.end())); .map(|(t, range)| (range.start(), t, range.end()));
python::TopParser::new() python::TopParser::new()
.parse(source, mode, lexer) .parse(source, mode, lexer)
.map_err(|e| parse_error_from_lalrpop(e, source_path)) .map_err(parse_error_from_lalrpop)
} }
fn parse_filtered_tokens( fn parse_filtered_tokens(
lxr: impl IntoIterator<Item = LexResult>, lxr: impl IntoIterator<Item = LexResult>,
source: &str, source: &str,
mode: Mode, mode: Mode,
source_path: &str,
) -> Result<Mod, ParseError> { ) -> Result<Mod, ParseError> {
let marker_token = (Tok::start_marker(mode), TextRange::default()); let marker_token = (Tok::start_marker(mode), TextRange::default());
let lexer = iter::once(Ok(marker_token)).chain(lxr); let lexer = std::iter::once(Ok(marker_token)).chain(lxr);
python::TopParser::new() python::TopParser::new()
.parse( .parse(
source, source,
mode, mode,
lexer.map_ok(|(t, range)| (range.start(), t, range.end())), lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
) )
.map_err(|e| parse_error_from_lalrpop(e, source_path)) .map_err(parse_error_from_lalrpop)
} }
/// Represents represent errors that occur during parsing and are /// Represents represent errors that occur during parsing and are
@ -277,7 +262,6 @@ fn parse_filtered_tokens(
pub struct ParseError { pub struct ParseError {
pub error: ParseErrorType, pub error: ParseErrorType,
pub offset: TextSize, pub offset: TextSize,
pub source_path: String,
} }
impl std::ops::Deref for ParseError { impl std::ops::Deref for ParseError {
@ -294,7 +278,7 @@ impl std::error::Error for ParseError {
} }
} }
impl fmt::Display for ParseError { impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!( write!(
f, f,
@ -324,28 +308,20 @@ pub enum ParseErrorType {
impl std::error::Error for ParseErrorType {} impl std::error::Error for ParseErrorType {}
// Convert `lalrpop_util::ParseError` to our internal type // Convert `lalrpop_util::ParseError` to our internal type
fn parse_error_from_lalrpop( fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> ParseError {
err: LalrpopError<TextSize, Tok, LexicalError>,
source_path: &str,
) -> ParseError {
let source_path = source_path.to_owned();
match err { match err {
// TODO: Are there cases where this isn't an EOF? // TODO: Are there cases where this isn't an EOF?
LalrpopError::InvalidToken { location } => ParseError { LalrpopError::InvalidToken { location } => ParseError {
error: ParseErrorType::Eof, error: ParseErrorType::Eof,
offset: location, offset: location,
source_path,
}, },
LalrpopError::ExtraToken { token } => ParseError { LalrpopError::ExtraToken { token } => ParseError {
error: ParseErrorType::ExtraToken(token.1), error: ParseErrorType::ExtraToken(token.1),
offset: token.0, offset: token.0,
source_path,
}, },
LalrpopError::User { error } => ParseError { LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error), error: ParseErrorType::Lexical(error.error),
offset: error.location, offset: error.location,
source_path,
}, },
LalrpopError::UnrecognizedToken { token, expected } => { LalrpopError::UnrecognizedToken { token, expected } => {
// Hacky, but it's how CPython does it. See PyParser_AddToken, // Hacky, but it's how CPython does it. See PyParser_AddToken,
@ -354,7 +330,6 @@ fn parse_error_from_lalrpop(
ParseError { ParseError {
error: ParseErrorType::UnrecognizedToken(token.1, expected), error: ParseErrorType::UnrecognizedToken(token.1, expected),
offset: token.0, offset: token.0,
source_path,
} }
} }
LalrpopError::UnrecognizedEof { location, expected } => { LalrpopError::UnrecognizedEof { location, expected } => {
@ -364,13 +339,11 @@ fn parse_error_from_lalrpop(
ParseError { ParseError {
error: ParseErrorType::Lexical(LexicalErrorType::IndentationError), error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
offset: location, offset: location,
source_path,
} }
} else { } else {
ParseError { ParseError {
error: ParseErrorType::Eof, error: ParseErrorType::Eof,
offset: location, offset: location,
source_path,
} }
} }
} }
@ -629,63 +602,63 @@ mod tests {
#[test] #[test]
fn test_parse_empty() { fn test_parse_empty() {
let parse_ast = parse_suite("", "<test>").unwrap(); let parse_ast = parse_suite("").unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_string() { fn test_parse_string() {
let source = "'Hello world'"; let source = "'Hello world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string() { fn test_parse_f_string() {
let source = "f'Hello world'"; let source = "f'Hello world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_print_hello() { fn test_parse_print_hello() {
let source = "print('Hello world')"; let source = "print('Hello world')";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_print_2() { fn test_parse_print_2() {
let source = "print('Hello world', 2)"; let source = "print('Hello world', 2)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_kwargs() { fn test_parse_kwargs() {
let source = "my_func('positional', keyword=2)"; let source = "my_func('positional', keyword=2)";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_if_elif_else() { fn test_parse_if_elif_else() {
let source = "if 1: 10\nelif 2: 20\nelse: 30"; let source = "if 1: 10\nelif 2: 20\nelse: 30";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_lambda() { fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_lambda_no_args() { fn test_parse_lambda_no_args() {
let source = "lambda: 1"; let source = "lambda: 1";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -693,7 +666,7 @@ mod tests {
fn test_parse_tuples() { fn test_parse_tuples() {
let source = "a, b = 4, 5"; let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -705,7 +678,7 @@ class Foo(A, B):
def method_with_default(self, arg='default'): def method_with_default(self, arg='default'):
pass pass
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -736,7 +709,7 @@ class Foo[**P](): ...
class Foo[X, Y: str, *U, **P](): class Foo[X, Y: str, *U, **P]():
pass pass
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
fn test_parse_function_definition() { fn test_parse_function_definition() {
@ -762,76 +735,76 @@ def func[**P](*args: P.args, **kwargs: P.kwargs):
def func[T, U: str, *Ts, **P](): def func[T, U: str, *Ts, **P]():
pass pass
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
fn test_parse_dict_comprehension() { fn test_parse_dict_comprehension() {
let source = "{x1: x2 for y in z}"; let source = "{x1: x2 for y in z}";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_list_comprehension() { fn test_parse_list_comprehension() {
let source = "[x for y in z]"; let source = "[x for y in z]";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_double_list_comprehension() { fn test_parse_double_list_comprehension() {
let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]"; let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_generator_comprehension() { fn test_parse_generator_comprehension() {
let source = "(x for y in z)"; let source = "(x for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_named_expression_generator_comprehension() { fn test_parse_named_expression_generator_comprehension() {
let source = "(x := y + 1 for y in z)"; let source = "(x := y + 1 for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_if_else_generator_comprehension() { fn test_parse_if_else_generator_comprehension() {
let source = "(x if y else y for y in z)"; let source = "(x if y else y for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_bool_op_or() { fn test_parse_bool_op_or() {
let source = "x or y"; let source = "x or y";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_bool_op_and() { fn test_parse_bool_op_and() {
let source = "x and y"; let source = "x and y";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_slice() { fn test_slice() {
let source = "x[1:2:3]"; let source = "x[1:2:3]";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_named_expression() { fn test_named_expression() {
let source = "(x := ( y * z ))"; let source = "(x := ( y * z ))";
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -865,7 +838,7 @@ with (0 as a,): pass
with (0 as a, 1 as b): pass with (0 as a, 1 as b): pass
with (0 as a, 1 as b,): pass with (0 as a, 1 as b,): pass
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -889,7 +862,7 @@ with (yield from a): pass
with ((yield)): pass with ((yield)): pass
with ((yield from a)): pass with ((yield from a)): pass
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -912,7 +885,7 @@ with ((yield from a)): pass
"with a := 0 as x: pass", "with a := 0 as x: pass",
"with (a := 0 as x): pass", "with (a := 0 as x): pass",
] { ] {
assert!(parse_suite(source, "<test>").is_err()); assert!(parse_suite(source).is_err());
} }
} }
@ -924,7 +897,7 @@ array[0, *indexes, -1] = array_slice
array[*indexes_to_select, *indexes_to_select] array[*indexes_to_select, *indexes_to_select]
array[3:5, *indexes_to_select] array[3:5, *indexes_to_select]
"; ";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -937,7 +910,7 @@ array[3:5, *indexes_to_select]
("OFFSET %d" % offset) if offset else None, ("OFFSET %d" % offset) if offset else None,
) )
)"#; )"#;
let parse_ast = parse_expression(source, "<test>").unwrap(); let parse_ast = parse_expression(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -950,7 +923,6 @@ except TypeError as e:
print(f'caught {type(e)}') print(f'caught {type(e)}')
except OSError as e: except OSError as e:
print(f'caught {type(e)}')", print(f'caught {type(e)}')",
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -966,7 +938,6 @@ except* TypeError as e:
print(f'caught {type(e)} with nested {e.exceptions}') print(f'caught {type(e)} with nested {e.exceptions}')
except* OSError as e: except* OSError as e:
print(f'caught {type(e)} with nested {e.exceptions}')"#, print(f'caught {type(e)} with nested {e.exceptions}')"#,
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -974,7 +945,7 @@ except* OSError as e:
#[test] #[test]
fn test_dict_unpacking() { fn test_dict_unpacking() {
let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap(); let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -982,8 +953,8 @@ except* OSError as e:
fn test_modes() { fn test_modes() {
let source = "a[0][1][2][3][4]"; let source = "a[0][1][2][3][4]";
assert!(parse(source, Mode::Expression, "<embedded>").is_ok()); assert!(parse(source, Mode::Expression).is_ok());
assert!(parse(source, Mode::Module, "<embedded>").is_ok()); assert!(parse(source, Mode::Module).is_ok());
} }
#[test] #[test]
@ -1030,7 +1001,7 @@ type X[T] \
type X = int; type X = str; type X = type type X = int; type X = str; type X = type
class X: type X = int class X: type X = int
"#; "#;
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -1068,13 +1039,13 @@ type = x = 1
x = type = 1 x = type = 1
lambda x: type lambda x: type
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
fn test_invalid_type() { fn test_invalid_type() {
assert!(parse_suite("a: type X = int", "<test>").is_err()); assert!(parse_suite("a: type X = int").is_err());
assert!(parse_suite("lambda: type X = int", "<test>").is_err()); assert!(parse_suite("lambda: type X = int").is_err());
} }
#[test] #[test]
@ -1099,7 +1070,7 @@ x = 10000
x = 133333 x = 133333
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -1125,7 +1096,7 @@ if 10 .real:
y = 100[no] y = 100[no]
y = 100(no) y = 100(no)
"; ";
assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -1153,7 +1124,7 @@ match match:
match = lambda query: query == event match = lambda query: query == event
print(match(12)) print(match(12))
"; ";
insta::assert_debug_snapshot!(parse_suite(source, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(source).unwrap());
} }
#[test] #[test]
@ -1323,7 +1294,7 @@ match w := x,:
case y as v,: case y as v,:
z = 0 z = 0
"#; "#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -1360,7 +1331,6 @@ match x, y,:
case z: case z:
pass pass
"#, "#,
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1375,7 +1345,6 @@ match x:
case f"{y}": case f"{y}":
pass pass
"#, "#,
"<test>",
) )
.err(); .err();
assert!( assert!(
@ -1390,7 +1359,6 @@ match x:
r" r"
def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
", ",
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1409,7 +1377,6 @@ class Abcd:
pass pass
" "
.trim(), .trim(),
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1486,7 +1453,6 @@ foo.bar[0].baz[2].egg??
" "
.trim(), .trim(),
Mode::Ipython, Mode::Ipython,
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1500,7 +1466,7 @@ a = 1
" "
.trim(); .trim();
let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default()); let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default());
let parse_err = parse_tokens(lxr, source, Mode::Module, "<test>").unwrap_err(); let parse_err = parse_tokens(lxr, source, Mode::Module).unwrap_err();
assert_eq!( assert_eq!(
parse_err.to_string(), parse_err.to_string(),
"IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6" "IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6"
@ -1540,7 +1506,6 @@ f"""{
}""" }"""
"# "#
.trim(), .trim(),
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1556,7 +1521,6 @@ u"foo" f"{bar}" "baz" " some"
u"foo" f"bar {baz} really" u"bar" "no" u"foo" f"bar {baz} really" u"bar" "no"
"# "#
.trim(), .trim(),
"<test>",
) )
.unwrap(); .unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
@ -1565,7 +1529,7 @@ u"foo" f"bar {baz} really" u"bar" "no"
#[test] #[test]
fn test_unicode_aliases() { fn test_unicode_aliases() {
// https://github.com/RustPython/RustPython/issues/4566 // https://github.com/RustPython/RustPython/issues/4566
let parse_ast = parse_suite(r#"x = "\N{BACKSPACE}another cool trick""#, "<test>").unwrap(); let parse_ast = parse_suite(r#"x = "\N{BACKSPACE}another cool trick""#).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
} }

84
crates/ruff_python_parser/src/string.rs
View file

@ -489,7 +489,7 @@ mod tests {
fn string_parser_escaped_eol(eol: &str) -> Suite { fn string_parser_escaped_eol(eol: &str) -> Suite {
let source = format!(r"'text \{eol}more text'"); let source = format!(r"'text \{eol}more text'");
parse_suite(&source, "<test>").unwrap() parse_suite(&source).unwrap()
} }
#[test] #[test]
@ -513,7 +513,7 @@ mod tests {
#[test] #[test]
fn test_parse_fstring() { fn test_parse_fstring() {
let source = r#"f"{a}{ b }{{foo}}""#; let source = r#"f"{a}{ b }{{foo}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -521,7 +521,7 @@ mod tests {
#[test] #[test]
fn test_parse_fstring_nested_spec() { fn test_parse_fstring_nested_spec() {
let source = r#"f"{foo:{spec}}""#; let source = r#"f"{foo:{spec}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -529,20 +529,20 @@ mod tests {
#[test] #[test]
fn test_parse_fstring_not_nested_spec() { fn test_parse_fstring_not_nested_spec() {
let source = r#"f"{foo:spec}""#; let source = r#"f"{foo:spec}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_empty_fstring() { fn test_parse_empty_fstring() {
insta::assert_debug_snapshot!(parse_suite(r#"f"""#, "<test>").unwrap()); insta::assert_debug_snapshot!(parse_suite(r#"f"""#,).unwrap());
} }
#[test] #[test]
fn test_fstring_parse_self_documenting_base() { fn test_fstring_parse_self_documenting_base() {
let source = r#"f"{user=}""#; let source = r#"f"{user=}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -550,7 +550,7 @@ mod tests {
#[test] #[test]
fn test_fstring_parse_self_documenting_base_more() { fn test_fstring_parse_self_documenting_base_more() {
let source = r#"f"mix {user=} with text and {second=}""#; let source = r#"f"mix {user=} with text and {second=}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -558,13 +558,13 @@ mod tests {
#[test] #[test]
fn test_fstring_parse_self_documenting_format() { fn test_fstring_parse_self_documenting_format() {
let source = r#"f"{user=:>10}""#; let source = r#"f"{user=:>10}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
fn parse_fstring_error(source: &str) -> FStringErrorType { fn parse_fstring_error(source: &str) -> FStringErrorType {
parse_suite(source, "<test>") parse_suite(source)
.map_err(|e| match e.error { .map_err(|e| match e.error {
ParseErrorType::Lexical(LexicalErrorType::FStringError(e)) => e, ParseErrorType::Lexical(LexicalErrorType::FStringError(e)) => e,
e => unreachable!("Expected FStringError: {:?}", e), e => unreachable!("Expected FStringError: {:?}", e),
@ -585,111 +585,111 @@ mod tests {
parse_fstring_error("f'{lambda x: {x}}'"), parse_fstring_error("f'{lambda x: {x}}'"),
LambdaWithoutParentheses LambdaWithoutParentheses
); );
assert!(parse_suite(r#"f"{class}""#, "<test>").is_err()); assert!(parse_suite(r#"f"{class}""#,).is_err());
} }
#[test] #[test]
fn test_parse_fstring_not_equals() { fn test_parse_fstring_not_equals() {
let source = r#"f"{1 != 2}""#; let source = r#"f"{1 != 2}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_fstring_equals() { fn test_parse_fstring_equals() {
let source = r#"f"{42 == 42}""#; let source = r#"f"{42 == 42}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_fstring_self_doc_prec_space() { fn test_parse_fstring_self_doc_prec_space() {
let source = r#"f"{x =}""#; let source = r#"f"{x =}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_fstring_self_doc_trailing_space() { fn test_parse_fstring_self_doc_trailing_space() {
let source = r#"f"{x= }""#; let source = r#"f"{x= }""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_fstring_yield_expr() { fn test_parse_fstring_yield_expr() {
let source = r#"f"{yield}""#; let source = r#"f"{yield}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_string_concat() { fn test_parse_string_concat() {
let source = "'Hello ' 'world'"; let source = "'Hello ' 'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_string_concat_1() { fn test_parse_u_string_concat_1() {
let source = "'Hello ' u'world'"; let source = "'Hello ' u'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_string_concat_2() { fn test_parse_u_string_concat_2() {
let source = "u'Hello ' 'world'"; let source = "u'Hello ' 'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_1() { fn test_parse_f_string_concat_1() {
let source = "'Hello ' f'world'"; let source = "'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_2() { fn test_parse_f_string_concat_2() {
let source = "'Hello ' f'world'"; let source = "'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_3() { fn test_parse_f_string_concat_3() {
let source = "'Hello ' f'world{\"!\"}'"; let source = "'Hello ' f'world{\"!\"}'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_f_string_concat_4() { fn test_parse_f_string_concat_4() {
let source = "'Hello ' f'world{\"!\"}' 'again!'"; let source = "'Hello ' f'world{\"!\"}' 'again!'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_f_string_concat_1() { fn test_parse_u_f_string_concat_1() {
let source = "u'Hello ' f'world'"; let source = "u'Hello ' f'world'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_u_f_string_concat_2() { fn test_parse_u_f_string_concat_2() {
let source = "u'Hello ' f'world' '!'"; let source = "u'Hello ' f'world' '!'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_string_triple_quotes_with_kind() { fn test_parse_string_triple_quotes_with_kind() {
let source = "u'''Hello, world!'''"; let source = "u'''Hello, world!'''";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -697,7 +697,7 @@ mod tests {
fn test_single_quoted_byte() { fn test_single_quoted_byte() {
// single quote // single quote
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -705,7 +705,7 @@ mod tests {
fn test_double_quoted_byte() { fn test_double_quoted_byte() {
// double quote // double quote
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -713,42 +713,42 @@ mod tests {
fn test_escape_char_in_byte_literal() { fn test_escape_char_in_byte_literal() {
// backslash does not escape // backslash does not escape
let source = r#"b"omkmok\Xaa""#; // spell-checker:ignore omkmok let source = r#"b"omkmok\Xaa""#; // spell-checker:ignore omkmok
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_byte_literal_1() { fn test_raw_byte_literal_1() {
let source = r"rb'\x1z'"; let source = r"rb'\x1z'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_byte_literal_2() { fn test_raw_byte_literal_2() {
let source = r"rb'\\'"; let source = r"rb'\\'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_escape_octet() { fn test_escape_octet() {
let source = r"b'\43a\4\1234'"; let source = r"b'\43a\4\1234'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_escaped_newline() { fn test_fstring_escaped_newline() {
let source = r#"f"\n{x}""#; let source = r#"f"\n{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_constant_range() { fn test_fstring_constant_range() {
let source = r#"f"aaa{bbb}ccc{ddd}eee""#; let source = r#"f"aaa{bbb}ccc{ddd}eee""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -756,28 +756,28 @@ mod tests {
fn test_fstring_unescaped_newline() { fn test_fstring_unescaped_newline() {
let source = r#"f""" let source = r#"f"""
{x}""""#; {x}""""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_fstring_escaped_character() { fn test_fstring_escaped_character() {
let source = r#"f"\\{x}""#; let source = r#"f"\\{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_raw_fstring() { fn test_raw_fstring() {
let source = r#"rf"{x}""#; let source = r#"rf"{x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_triple_quoted_raw_fstring() { fn test_triple_quoted_raw_fstring() {
let source = r#"rf"""{x}""""#; let source = r#"rf"""{x}""""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -785,14 +785,14 @@ mod tests {
fn test_fstring_line_continuation() { fn test_fstring_line_continuation() {
let source = r#"rf"\ let source = r#"rf"\
{x}""#; {x}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test] #[test]
fn test_parse_fstring_nested_string_spec() { fn test_parse_fstring_nested_string_spec() {
let source = r#"f"{foo:{''}}""#; let source = r#"f"{foo:{''}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -800,7 +800,7 @@ mod tests {
#[test] #[test]
fn test_parse_fstring_nested_concatenation_string_spec() { fn test_parse_fstring_nested_concatenation_string_spec() {
let source = r#"f"{foo:{'' ''}}""#; let source = r#"f"{foo:{'' ''}}""#;
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -809,7 +809,7 @@ mod tests {
#[test] #[test]
fn test_dont_panic_on_8_in_octal_escape() { fn test_dont_panic_on_8_in_octal_escape() {
let source = r"bold = '\038[1m'"; let source = r"bold = '\038[1m'";
let parse_ast = parse_suite(source, "<test>").unwrap(); let parse_ast = parse_suite(source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
@ -820,7 +820,7 @@ mod tests {
#[test] #[test]
fn $name() { fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias); let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = parse_suite(&source, "<test>").unwrap(); let parse_ast = parse_suite(&source).unwrap();
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
)* )*

8
crates/ruff_python_parser/src/typing.rs
View file

@ -33,15 +33,11 @@ pub fn parse_type_annotation(
// isn't the case, e.g., for implicit concatenations, or for annotations that contain // isn't the case, e.g., for implicit concatenations, or for annotations that contain
// escaped quotes. // escaped quotes.
let leading_quote = str::leading_quote(expression).unwrap(); let leading_quote = str::leading_quote(expression).unwrap();
let expr = parse_expression_starts_at( let expr = parse_expression_starts_at(value, range.start() + leading_quote.text_len())?;
value,
"<filename>",
range.start() + leading_quote.text_len(),
)?;
Ok((expr, AnnotationKind::Simple)) Ok((expr, AnnotationKind::Simple))
} else { } else {
// Otherwise, consider this a "complex" annotation. // Otherwise, consider this a "complex" annotation.
let mut expr = parse_expression(value, "<filename>")?; let mut expr = parse_expression(value)?;
relocate_expr(&mut expr, range); relocate_expr(&mut expr, range);
Ok((expr, AnnotationKind::Complex)) Ok((expr, AnnotationKind::Complex))
} }

2
crates/ruff_python_semantic/src/analyze/type_inference.rs
View file

@ -436,7 +436,7 @@ mod tests {
use crate::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType}; use crate::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType};
fn parse(expression: &str) -> Expr { fn parse(expression: &str) -> Expr {
parse_expression(expression, "").unwrap() parse_expression(expression).unwrap()
} }
#[test] #[test]

10
crates/ruff_python_trivia/src/whitespace.rs
View file

@ -91,25 +91,25 @@ mod tests {
#[test] #[test]
fn trailing_content() -> Result<(), ParseError> { fn trailing_content() -> Result<(), ParseError> {
let contents = "x = 1"; let contents = "x = 1";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1; y = 2"; let contents = "x = 1; y = 2";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(has_trailing_content(stmt.end(), &locator)); assert!(has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 "; let contents = "x = 1 ";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
let contents = "x = 1 # Comment"; let contents = "x = 1 # Comment";
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));
@ -119,7 +119,7 @@ x = 1
y = 2 y = 2
" "
.trim(); .trim();
let program = parse_suite(contents, "<filename>")?; let program = parse_suite(contents)?;
let stmt = program.first().unwrap(); let stmt = program.first().unwrap();
let locator = Locator::new(contents); let locator = Locator::new(contents);
assert!(!has_trailing_content(stmt.end(), &locator)); assert!(!has_trailing_content(stmt.end(), &locator));

2
crates/ruff_shrinking/src/main.rs
View file

@ -320,7 +320,7 @@ fn minimization_step(
last_strategy_and_idx: Option<(&'static dyn Strategy, usize)>, last_strategy_and_idx: Option<(&'static dyn Strategy, usize)>,
) -> Result<Option<(&'static dyn Strategy, usize, String)>> { ) -> Result<Option<(&'static dyn Strategy, usize, String)>> {
let tokens = ruff_python_parser::tokenize(input, Mode::Module); let tokens = ruff_python_parser::tokenize(input, Mode::Module);
let ast = ruff_python_parser::parse_program_tokens(tokens, input, "input.py", false) let ast = ruff_python_parser::parse_program_tokens(tokens, input, false)
.context("not valid python")?; .context("not valid python")?;
// Try the last succeeding strategy first, skipping all that failed last time // Try the last succeeding strategy first, skipping all that failed last time

12
crates/ruff_wasm/src/lib.rs
View file

@ -248,7 +248,7 @@ impl Workspace {
/// Parses the content and returns its AST /// Parses the content and returns its AST
pub fn parse(&self, contents: &str) -> Result<String, Error> { pub fn parse(&self, contents: &str) -> Result<String, Error> {
let parsed = ruff_python_parser::parse(contents, Mode::Module, ".").map_err(into_error)?; let parsed = ruff_python_parser::parse(contents, Mode::Module).map_err(into_error)?;
Ok(format!("{parsed:#?}")) Ok(format!("{parsed:#?}"))
} }
@ -271,20 +271,20 @@ struct ParsedModule<'a> {
} }
impl<'a> ParsedModule<'a> { impl<'a> ParsedModule<'a> {
fn from_source(source: &'a str) -> Result<Self, Error> { fn from_source(source_code: &'a str) -> Result<Self, Error> {
let tokens: Vec<_> = ruff_python_parser::lexer::lex(source, Mode::Module).collect(); let tokens: Vec<_> = ruff_python_parser::lexer::lex(source_code, Mode::Module).collect();
let mut comment_ranges = CommentRangesBuilder::default(); let mut comment_ranges = CommentRangesBuilder::default();
for (token, range) in tokens.iter().flatten() { for (token, range) in tokens.iter().flatten() {
comment_ranges.visit_token(token, *range); comment_ranges.visit_token(token, *range);
} }
let comment_ranges = comment_ranges.finish(); let comment_ranges = comment_ranges.finish();
let module = parse_tokens(tokens, source, Mode::Module, ".").map_err(into_error)?; let module = parse_tokens(tokens, source_code, Mode::Module).map_err(into_error)?;
Ok(Self { Ok(Self {
source_code: source, source_code,
comment_ranges,
module, module,
comment_ranges,
}) })
} }

7
fuzz/fuzz_targets/ruff_parse_idempotency.rs
View file

@ -13,9 +13,9 @@ fn do_fuzz(case: &[u8]) -> Corpus {
}; };
// round trip it once to get a formatted version // round trip it once to get a formatted version
if let Ok(first) = round_trip(code, "fuzzed-source.py") { if let Ok(first) = round_trip(code) {
// round trip it a second time to get a case to compare against // round trip it a second time to get a case to compare against
if let Ok(second) = round_trip(&first, "fuzzed-source.py") { if let Ok(second) = round_trip(&first) {
if cfg!(feature = "full-idempotency") { if cfg!(feature = "full-idempotency") {
// potentially, we don't want to test for full idempotency, but just for unsteady states // potentially, we don't want to test for full idempotency, but just for unsteady states
// enable the "full-idempotency" feature when fuzzing for full idempotency // enable the "full-idempotency" feature when fuzzing for full idempotency
@ -31,8 +31,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
} else if first != second { } else if first != second {
// by the third time we've round-tripped it, we shouldn't be introducing any more // by the third time we've round-tripped it, we shouldn't be introducing any more
// changes; if we do, then it's likely that we're in an unsteady parsing state // changes; if we do, then it's likely that we're in an unsteady parsing state
let third = round_trip(&second, "fuzzed-source.py") let third = round_trip(&second).expect("Couldn't round-trip the processed source.");
.expect("Couldn't round-trip the processed source.");
let diff = TextDiff::from_lines(&second, &third) let diff = TextDiff::from_lines(&second, &third)
.unified_diff() .unified_diff()
.header("Parsed twice", "Parsed three times") .header("Parsed twice", "Parsed three times")

2
fuzz/fuzz_targets/ruff_parse_simple.rs
View file

@ -15,7 +15,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
// just round-trip it once to trigger both parse and unparse // just round-trip it once to trigger both parse and unparse
let locator = Locator::new(code); let locator = Locator::new(code);
let python_ast = match parse_suite(code, "fuzzed-source.py") { let python_ast = match parse_suite(code) {
Ok(stmts) => stmts, Ok(stmts) => stmts,
Err(ParseError { offset, .. }) => { Err(ParseError { offset, .. }) => {
let offset = offset.to_usize(); let offset = offset.to_usize();