mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-30 15:17:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Pull in RustPython parser (#6099)

Browse source
This commit is contained in:
Micha Reiser 2023-07-27 11:29:11 +02:00 • committed by GitHub
parent 86539c1fc5
commit 40f54375cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
779 changed files with 108400 additions and 2078 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/ruff_python_ast/src/all.rs
View file

@ -1,5 +1,5 @@
use crate::{self as ast, Constant, Expr, Stmt};
use bitflags::bitflags;
use rustpython_ast::{self as ast, Constant, Expr, Stmt};
bitflags! {
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)]

20
crates/ruff_python_ast/src/call_path.rs
View file

@ -1,4 +1,4 @@
use rustpython_ast::{self as ast, Expr};
use crate::{nodes, Expr};
use smallvec::{smallvec, SmallVec};
/// A representation of a qualified name, like `typing.List`.
@ -11,14 +11,16 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr1 = match expr {
Expr::Attribute(attr1) => attr1,
// Ex) `foo`
Expr::Name(ast::ExprName { id, .. }) => return Some(CallPath::from_slice(&[id.as_str()])),
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[id.as_str()]))
}
_ => return None,
};
let attr2 = match attr1.value.as_ref() {
Expr::Attribute(attr2) => attr2,
// Ex) `foo.bar`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[id.as_str(), attr1.attr.as_str()]))
}
_ => return None,
@ -27,7 +29,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr3 = match attr2.value.as_ref() {
Expr::Attribute(attr3) => attr3,
// Ex) `foo.bar.baz`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr2.attr.as_str(),
@ -40,7 +42,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr4 = match attr3.value.as_ref() {
Expr::Attribute(attr4) => attr4,
// Ex) `foo.bar.baz.bop`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr3.attr.as_str(),
@ -54,7 +56,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr5 = match attr4.value.as_ref() {
Expr::Attribute(attr5) => attr5,
// Ex) `foo.bar.baz.bop.bap`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr4.attr.as_str(),
@ -69,7 +71,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr6 = match attr5.value.as_ref() {
Expr::Attribute(attr6) => attr6,
// Ex) `foo.bar.baz.bop.bap.bab`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr5.attr.as_str(),
@ -85,7 +87,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr7 = match attr6.value.as_ref() {
Expr::Attribute(attr7) => attr7,
// Ex) `foo.bar.baz.bop.bap.bab.bob`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr6.attr.as_str(),
@ -102,7 +104,7 @@ pub fn collect_call_path(expr: &Expr) -> Option<CallPath> {
let attr8 = match attr7.value.as_ref() {
Expr::Attribute(attr8) => attr8,
// Ex) `foo.bar.baz.bop.bap.bab.bob.bib`
Expr::Name(ast::ExprName { id, .. }) => {
Expr::Name(nodes::ExprName { id, .. }) => {
return Some(CallPath::from_slice(&[
id.as_str(),
attr7.attr.as_str(),

10
crates/ruff_python_ast/src/cast.rs
View file

@ -1,17 +1,17 @@
use rustpython_ast::{self as ast, Decorator, Stmt};
use crate::{nodes, Decorator, Stmt};
pub fn name(stmt: &Stmt) -> &str {
match stmt {
Stmt::FunctionDef(ast::StmtFunctionDef { name, .. })
| Stmt::AsyncFunctionDef(ast::StmtAsyncFunctionDef { name, .. }) => name.as_str(),
Stmt::FunctionDef(nodes::StmtFunctionDef { name, .. })
| Stmt::AsyncFunctionDef(nodes::StmtAsyncFunctionDef { name, .. }) => name.as_str(),
_ => panic!("Expected Stmt::FunctionDef | Stmt::AsyncFunctionDef"),
}
}
pub fn decorator_list(stmt: &Stmt) -> &[Decorator] {
match stmt {
Stmt::FunctionDef(ast::StmtFunctionDef { decorator_list, .. })
| Stmt::AsyncFunctionDef(ast::StmtAsyncFunctionDef { decorator_list, .. }) => {
Stmt::FunctionDef(nodes::StmtFunctionDef { decorator_list, .. })
| Stmt::AsyncFunctionDef(nodes::StmtAsyncFunctionDef { decorator_list, .. }) => {
decorator_list
}
_ => panic!("Expected Stmt::FunctionDef | Stmt::AsyncFunctionDef"),

2
crates/ruff_python_ast/src/comparable.rs
View file

@ -1,8 +1,8 @@
//! An equivalent object hierarchy to the `RustPython` AST hierarchy, but with the
//! ability to compare expressions for equality (via [`Eq`] and [`Hash`]).
use crate as ast;
use num_bigint::BigInt;
use rustpython_ast as ast;
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
pub enum ComparableExprContext {

4
crates/ruff_python_ast/src/function.rs
View file

@ -1,8 +1,8 @@
use crate::node::AnyNodeRef;
use ruff_text_size::TextRange;
use rustpython_ast::{
use crate::{
Arguments, Decorator, Expr, Identifier, Ranged, StmtAsyncFunctionDef, StmtFunctionDef, Suite,
};
use ruff_text_size::TextRange;
/// Enum that represents any python function definition.
#[derive(Copy, Clone, PartialEq, Debug)]

2
crates/ruff_python_ast/src/hashable.rs
View file

@ -1,6 +1,6 @@
use std::hash::Hash;
use rustpython_ast::Expr;
use crate::Expr;
use crate::comparable::ComparableExpr;

12
crates/ruff_python_ast/src/helpers.rs
View file

@ -1,12 +1,12 @@
use std::borrow::Cow;
use std::path::Path;
use num_traits::Zero;
use ruff_text_size::TextRange;
use rustpython_ast::{
use crate::{
self as ast, Arguments, Constant, ExceptHandler, Expr, Keyword, MatchCase, Pattern, Ranged,
Stmt, TypeParam,
};
use num_traits::Zero;
use ruff_text_size::TextRange;
use smallvec::SmallVec;
use crate::call_path::CallPath;
@ -1288,11 +1288,11 @@ mod tests {
use std::cell::RefCell;
use std::vec;
use ruff_text_size::TextRange;
use rustpython_ast::{
self, Constant, Expr, ExprConstant, ExprContext, ExprName, Identifier, Stmt, StmtTypeAlias,
use crate::{
Constant, Expr, ExprConstant, ExprContext, ExprName, Identifier, Stmt, StmtTypeAlias,
TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple,
};
use ruff_text_size::TextRange;
use crate::helpers::{any_over_stmt, any_over_type_param, resolve_imported_module_path};

38
crates/ruff_python_ast/src/identifier.rs
View file

@ -10,8 +10,8 @@
//!
//! This module can be used to identify the [`TextRange`] of the `except` token.
use crate::{self as ast, Alias, Arg, ArgWithDefault, ExceptHandler, Ranged, Stmt};
use ruff_text_size::{TextLen, TextRange, TextSize};
use rustpython_ast::{self as ast, Alias, Arg, ArgWithDefault, ExceptHandler, Ranged, Stmt};
use ruff_python_trivia::{is_python_whitespace, Cursor};
@ -200,38 +200,17 @@ impl Iterator for IdentifierTokenizer<'_> {
#[cfg(test)]
mod tests {
use ruff_text_size::{TextRange, TextSize};
use rustpython_ast::{Ranged, Stmt};
use rustpython_parser::{Parse, ParseError};
use crate::identifier;
use crate::identifier::IdentifierTokenizer;
use super::IdentifierTokenizer;
use ruff_text_size::{TextLen, TextRange, TextSize};
#[test]
fn extract_else_range() -> Result<(), ParseError> {
let contents = r#"
for x in y:
pass
else:
pass
"#
.trim();
let stmt = Stmt::parse(contents, "<filename>")?;
let range = identifier::else_(&stmt, contents).unwrap();
assert_eq!(&contents[range], "else");
assert_eq!(
range,
TextRange::new(TextSize::from(21), TextSize::from(25))
);
Ok(())
}
#[test]
fn extract_global_names() -> Result<(), ParseError> {
fn extract_global_names() {
let contents = r#"global X,Y, Z"#.trim();
let stmt = Stmt::parse(contents, "<filename>")?;
let mut names = IdentifierTokenizer::new(contents, stmt.range());
let mut names = IdentifierTokenizer::new(
contents,
TextRange::new(TextSize::new(0), contents.text_len()),
);
let range = names.next_token().unwrap();
assert_eq!(&contents[range], "global");
@ -251,6 +230,5 @@ else:
range,
TextRange::new(TextSize::from(12), TextSize::from(13))
);
Ok(())
}
}

32
crates/ruff_python_ast/src/lib.rs
View file

@ -1,3 +1,5 @@
use ruff_text_size::{TextRange, TextSize};
pub mod all;
pub mod call_path;
pub mod cast;
@ -9,6 +11,7 @@ pub mod helpers;
pub mod identifier;
pub mod imports;
pub mod node;
mod nodes;
pub mod relocate;
pub mod statement_visitor;
pub mod stmt_if;
@ -17,3 +20,32 @@ pub mod traversal;
pub mod types;
pub mod visitor;
pub mod whitespace;
pub use nodes::*;
pub trait Ranged {
fn range(&self) -> TextRange;
fn start(&self) -> TextSize {
self.range().start()
}
fn end(&self) -> TextSize {
self.range().end()
}
}
impl Ranged for TextRange {
fn range(&self) -> TextRange {
*self
}
}
impl<T> Ranged for &T
where
T: Ranged,
{
fn range(&self) -> TextRange {
T::range(self)
}
}

4
crates/ruff_python_ast/src/node.rs
View file

@ -1,9 +1,9 @@
use ruff_text_size::TextRange;
use rustpython_ast::{
use crate::{
self as ast, Alias, Arg, ArgWithDefault, Arguments, Comprehension, Decorator, ExceptHandler,
Expr, Keyword, MatchCase, Mod, Pattern, Ranged, Stmt, TypeIgnore, TypeParam,
TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, WithItem,
};
use ruff_text_size::TextRange;
use std::ptr::NonNull;
pub trait AstNode: Ranged {

3037
crates/ruff_python_ast/src/nodes.rs Normal file
View file

File diff suppressed because it is too large Load diff

58
crates/ruff_python_ast/src/relocate.rs
View file

@ -1,5 +1,5 @@
use crate::{nodes, Expr, Keyword};
use ruff_text_size::TextRange;
use rustpython_ast::{self as ast, Expr, Keyword};
fn relocate_keyword(keyword: &mut Keyword, location: TextRange) {
relocate_expr(&mut keyword.value, location);
@ -9,13 +9,13 @@ fn relocate_keyword(keyword: &mut Keyword, location: TextRange) {
/// location.
pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
match expr {
Expr::BoolOp(ast::ExprBoolOp { values, range, .. }) => {
Expr::BoolOp(nodes::ExprBoolOp { values, range, .. }) => {
*range = location;
for expr in values {
relocate_expr(expr, location);
}
}
Expr::NamedExpr(ast::ExprNamedExpr {
Expr::NamedExpr(nodes::ExprNamedExpr {
target,
value,
range,
@ -24,22 +24,22 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(target, location);
relocate_expr(value, location);
}
Expr::BinOp(ast::ExprBinOp {
Expr::BinOp(nodes::ExprBinOp {
left, right, range, ..
}) => {
*range = location;
relocate_expr(left, location);
relocate_expr(right, location);
}
Expr::UnaryOp(ast::ExprUnaryOp { operand, range, .. }) => {
Expr::UnaryOp(nodes::ExprUnaryOp { operand, range, .. }) => {
*range = location;
relocate_expr(operand, location);
}
Expr::Lambda(ast::ExprLambda { body, range, .. }) => {
Expr::Lambda(nodes::ExprLambda { body, range, .. }) => {
*range = location;
relocate_expr(body, location);
}
Expr::IfExp(ast::ExprIfExp {
Expr::IfExp(nodes::ExprIfExp {
test,
body,
orelse,
@ -50,7 +50,7 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(body, location);
relocate_expr(orelse, location);
}
Expr::Dict(ast::ExprDict {
Expr::Dict(nodes::ExprDict {
keys,
values,
range,
@ -63,46 +63,46 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(expr, location);
}
}
Expr::Set(ast::ExprSet { elts, range }) => {
Expr::Set(nodes::ExprSet { elts, range }) => {
*range = location;
for expr in elts {
relocate_expr(expr, location);
}
}
Expr::ListComp(ast::ExprListComp { elt, range, .. }) => {
Expr::ListComp(nodes::ExprListComp { elt, range, .. }) => {
*range = location;
relocate_expr(elt, location);
}
Expr::SetComp(ast::ExprSetComp { elt, range, .. }) => {
Expr::SetComp(nodes::ExprSetComp { elt, range, .. }) => {
*range = location;
relocate_expr(elt, location);
}
Expr::DictComp(ast::ExprDictComp {
Expr::DictComp(nodes::ExprDictComp {
key, value, range, ..
}) => {
*range = location;
relocate_expr(key, location);
relocate_expr(value, location);
}
Expr::GeneratorExp(ast::ExprGeneratorExp { elt, range, .. }) => {
Expr::GeneratorExp(nodes::ExprGeneratorExp { elt, range, .. }) => {
*range = location;
relocate_expr(elt, location);
}
Expr::Await(ast::ExprAwait { value, range }) => {
Expr::Await(nodes::ExprAwait { value, range }) => {
*range = location;
relocate_expr(value, location);
}
Expr::Yield(ast::ExprYield { value, range }) => {
Expr::Yield(nodes::ExprYield { value, range }) => {
*range = location;
if let Some(expr) = value {
relocate_expr(expr, location);
}
}
Expr::YieldFrom(ast::ExprYieldFrom { value, range }) => {
Expr::YieldFrom(nodes::ExprYieldFrom { value, range }) => {
*range = location;
relocate_expr(value, location);
}
Expr::Compare(ast::ExprCompare {
Expr::Compare(nodes::ExprCompare {
left,
comparators,
range,
@ -114,7 +114,7 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(expr, location);
}
}
Expr::Call(ast::ExprCall {
Expr::Call(nodes::ExprCall {
func,
args,
keywords,
@ -129,7 +129,7 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_keyword(keyword, location);
}
}
Expr::FormattedValue(ast::ExprFormattedValue {
Expr::FormattedValue(nodes::ExprFormattedValue {
value,
format_spec,
range,
@ -141,20 +141,20 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(expr, location);
}
}
Expr::JoinedStr(ast::ExprJoinedStr { values, range }) => {
Expr::JoinedStr(nodes::ExprJoinedStr { values, range }) => {
*range = location;
for expr in values {
relocate_expr(expr, location);
}
}
Expr::Constant(ast::ExprConstant { range, .. }) => {
Expr::Constant(nodes::ExprConstant { range, .. }) => {
*range = location;
}
Expr::Attribute(ast::ExprAttribute { value, range, .. }) => {
Expr::Attribute(nodes::ExprAttribute { value, range, .. }) => {
*range = location;
relocate_expr(value, location);
}
Expr::Subscript(ast::ExprSubscript {
Expr::Subscript(nodes::ExprSubscript {
value,
slice,
range,
@ -164,26 +164,26 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(value, location);
relocate_expr(slice, location);
}
Expr::Starred(ast::ExprStarred { value, range, .. }) => {
Expr::Starred(nodes::ExprStarred { value, range, .. }) => {
*range = location;
relocate_expr(value, location);
}
Expr::Name(ast::ExprName { range, .. }) => {
Expr::Name(nodes::ExprName { range, .. }) => {
*range = location;
}
Expr::List(ast::ExprList { elts, range, .. }) => {
Expr::List(nodes::ExprList { elts, range, .. }) => {
*range = location;
for expr in elts {
relocate_expr(expr, location);
}
}
Expr::Tuple(ast::ExprTuple { elts, range, .. }) => {
Expr::Tuple(nodes::ExprTuple { elts, range, .. }) => {
*range = location;
for expr in elts {
relocate_expr(expr, location);
}
}
Expr::Slice(ast::ExprSlice {
Expr::Slice(nodes::ExprSlice {
lower,
upper,
step,
@ -200,7 +200,7 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
relocate_expr(expr, location);
}
}
Expr::LineMagic(ast::ExprLineMagic { range, .. }) => {
Expr::LineMagic(nodes::ExprLineMagic { range, .. }) => {
*range = location;
}
}

13
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__class_type_parameters.snap
View file

@ -1,13 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtClassDef
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- StmtExpr
- ExprConstant

12
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__compare.snap
View file

@ -1,12 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtExpr
- ExprCompare
- ExprConstant
- Lt
- Lt
- ExprName
- ExprConstant

12
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__decorators.snap
View file

@ -1,12 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtFunctionDef
- ExprName
- Arguments
- StmtPass
- StmtClassDef
- ExprName
- StmtPass

15
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__dict_comprehension.snap
View file

@ -1,15 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtExpr
- ExprDictComp
- Comprehension
- ExprName
- ExprName
- ExprName
- ExprBinOp
- ExprName
- Pow
- ExprConstant

19
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__function_arguments.snap
View file

@ -1,19 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtFunctionDef
- Arguments
- ExprConstant
- ExprConstant
- ExprConstant
- Arg
- Arg
- Arg
- Arg
- Arg
- Arg
- Arg
- Arg
- StmtPass

14
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__function_positional_only_with_default.snap
View file

@ -1,14 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtFunctionDef
- Arguments
- ExprConstant
- ExprConstant
- Arg
- Arg
- Arg
- Arg
- StmtPass

14
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__function_type_parameters.snap
View file

@ -1,14 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtFunctionDef
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- Arguments
- StmtExpr
- ExprConstant

11
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__list_comprehension.snap
View file

@ -1,11 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtExpr
- ExprListComp
- Comprehension
- ExprName
- ExprName
- ExprName

27
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__match_class_pattern.snap
View file

@ -1,27 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtMatch
- ExprName
- MatchCase
- PatternMatchClass
- ExprName
- PatternMatchValue
- ExprConstant
- PatternMatchValue
- ExprConstant
- StmtExpr
- ExprConstant
- MatchCase
- PatternMatchClass
- ExprName
- PatternMatchValue
- ExprConstant
- PatternMatchValue
- ExprConstant
- PatternMatchValue
- ExprConstant
- StmtExpr
- ExprConstant

11
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__set_comprehension.snap
View file

@ -1,11 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtExpr
- ExprSetComp
- Comprehension
- ExprName
- ExprName
- ExprName

15
crates/ruff_python_ast/src/snapshots/ruff_python_ast__visitor__tests__type_aliases.snap
View file

@ -1,15 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor.rs
expression: trace
---
- StmtTypeAlias
- ExprSubscript
- ExprName
- ExprName
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- ExprName

2
crates/ruff_python_ast/src/statement_visitor.rs
View file

@ -1,6 +1,6 @@
//! Specialized AST visitor trait and walk functions that only visit statements.
use rustpython_ast::{self as ast, ElifElseClause, ExceptHandler, MatchCase, Stmt};
use crate::{self as ast, ElifElseClause, ExceptHandler, MatchCase, Stmt};
/// A trait for AST visitors that only need to visit statements.
pub trait StatementVisitor<'a> {

36
crates/ruff_python_ast/src/stmt_if.rs
View file

@ -1,6 +1,6 @@
use crate::{ElifElseClause, Expr, Ranged, Stmt, StmtIf};
use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
use ruff_text_size::TextRange;
use rustpython_ast::{ElifElseClause, Expr, Ranged, Stmt, StmtIf};
use std::iter;
/// Return the `Range` of the first `Elif` or `Else` token in an `If` statement.
@ -42,36 +42,4 @@ pub fn if_elif_branches(stmt_if: &StmtIf) -> impl Iterator<Item = IfElifBranch>
}
#[cfg(test)]
mod test {
use crate::stmt_if::elif_else_range;
use ruff_text_size::TextSize;
use rustpython_ast::Stmt;
use rustpython_parser::{Parse, ParseError};
#[test]
fn extract_elif_else_range() -> Result<(), ParseError> {
let contents = "if a:
...
elif b:
...
";
let stmt = Stmt::parse(contents, "<filename>")?;
let stmt = Stmt::as_if_stmt(&stmt).unwrap();
let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
assert_eq!(range.start(), TextSize::from(14));
assert_eq!(range.end(), TextSize::from(18));
let contents = "if a:
...
else:
...
";
let stmt = Stmt::parse(contents, "<filename>")?;
let stmt = Stmt::as_if_stmt(&stmt).unwrap();
let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
assert_eq!(range.start(), TextSize::from(14));
assert_eq!(range.end(), TextSize::from(18));
Ok(())
}
}
mod test {}

2
crates/ruff_python_ast/src/traversal.rs
View file

@ -1,5 +1,5 @@
//! Utilities for manually traversing a Python AST.
use rustpython_ast::{self as ast, ExceptHandler, Stmt, Suite};
use crate::{self as ast, ExceptHandler, Stmt, Suite};
/// Given a [`Stmt`] and its parent, return the [`Suite`] that contains the [`Stmt`].
pub fn suite<'a>(stmt: &'a Stmt, parent: &'a Stmt) -> Option<&'a Suite> {

2
crates/ruff_python_ast/src/types.rs
View file

@ -1,6 +1,6 @@
use std::ops::Deref;
use rustpython_ast::{Expr, Stmt};
use crate::{Expr, Stmt};
#[derive(Clone)]
pub enum Node<'a> {

305
crates/ruff_python_ast/src/visitor.rs
View file

@ -2,7 +2,7 @@
pub mod preorder;
use rustpython_ast::{
use crate::{
self as ast, Alias, Arg, Arguments, BoolOp, CmpOp, Comprehension, Decorator, ElifElseClause,
ExceptHandler, Expr, ExprContext, Keyword, MatchCase, Operator, Pattern, Stmt, TypeParam,
TypeParamTypeVar, UnaryOp, WithItem,
@ -796,306 +796,3 @@ pub fn walk_cmp_op<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, cmp_op: &'a Cmp
#[allow(unused_variables)]
pub fn walk_alias<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, alias: &'a Alias) {}
#[cfg(test)]
mod tests {
use std::fmt::{Debug, Write};
use insta::assert_snapshot;
use rustpython_ast as ast;
use rustpython_parser::lexer::lex;
use rustpython_parser::{parse_tokens, Mode};
use crate::node::AnyNodeRef;
use crate::visitor::{
walk_alias, walk_arg, walk_arguments, walk_comprehension, walk_except_handler, walk_expr,
walk_keyword, walk_match_case, walk_pattern, walk_stmt, walk_type_param, walk_with_item,
Alias, Arg, Arguments, BoolOp, CmpOp, Comprehension, ExceptHandler, Expr, Keyword,
MatchCase, Operator, Pattern, Stmt, TypeParam, UnaryOp, Visitor, WithItem,
};
#[test]
fn function_arguments() {
let source = r#"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_positional_only_with_default() {
let source = r#"def a(b, c = 34,/, e = 20, *args): pass"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn compare() {
let source = r#"4 < x < 5"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn list_comprehension() {
let source = "[x for x in numbers]";
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn dict_comprehension() {
let source = "{x: x**2 for x in numbers}";
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn set_comprehension() {
let source = "{x for x in numbers}";
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn match_class_pattern() {
let source = r#"
match x:
case Point2D(0, 0):
...
case Point3D(x=0, y=0, z=0):
...
"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn decorators() {
let source = r#"
@decorator
def a():
pass
@test
class A:
pass
"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn type_aliases() {
let source = r#"type X[T: str, U, *Ts, **P] = list[T]"#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn class_type_parameters() {
let source = r#"class X[T: str, U, *Ts, **P]: ..."#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_type_parameters() {
let source = r#"def X[T: str, U, *Ts, **P](): ..."#;
let trace = trace_visitation(source);
assert_snapshot!(trace);
}
fn trace_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, Mode::Module, "test.py").unwrap();
let mut visitor = RecordVisitor::default();
walk_module(&mut visitor, &parsed);
visitor.output
}
fn walk_module<'a, V>(visitor: &mut V, module: &'a ast::Mod)
where
V: Visitor<'a> + ?Sized,
{
match module {
ast::Mod::Module(ast::ModModule {
body,
range: _,
type_ignores: _,
}) => {
visitor.visit_body(body);
}
ast::Mod::Interactive(ast::ModInteractive { body, range: _ }) => {
visitor.visit_body(body);
}
ast::Mod::Expression(ast::ModExpression { body, range: _ }) => visitor.visit_expr(body),
ast::Mod::FunctionType(ast::ModFunctionType {
range: _,
argtypes,
returns,
}) => {
for arg_type in argtypes {
visitor.visit_expr(arg_type);
}
visitor.visit_expr(returns);
}
}
}
/// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
/// and leafs for attributes.
#[derive(Default)]
struct RecordVisitor {
depth: usize,
output: String,
}
impl RecordVisitor {
fn enter_node<'a, T>(&mut self, node: T)
where
T: Into<AnyNodeRef<'a>>,
{
self.emit(&node.into().kind());
self.depth += 1;
}
fn exit_node(&mut self) {
self.depth -= 1;
}
fn emit(&mut self, text: &dyn Debug) {
for _ in 0..self.depth {
self.output.push_str(" ");
}
writeln!(self.output, "- {text:?}").unwrap();
}
}
impl Visitor<'_> for RecordVisitor {
fn visit_stmt(&mut self, stmt: &Stmt) {
self.enter_node(stmt);
walk_stmt(self, stmt);
self.exit_node();
}
fn visit_annotation(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_expr(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_bool_op(&mut self, bool_op: &BoolOp) {
self.emit(&bool_op);
}
fn visit_operator(&mut self, operator: &Operator) {
self.emit(&operator);
}
fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
self.emit(&unary_op);
}
fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
self.emit(&cmp_op);
}
fn visit_comprehension(&mut self, comprehension: &Comprehension) {
self.enter_node(comprehension);
walk_comprehension(self, comprehension);
self.exit_node();
}
fn visit_except_handler(&mut self, except_handler: &ExceptHandler) {
self.enter_node(except_handler);
walk_except_handler(self, except_handler);
self.exit_node();
}
fn visit_format_spec(&mut self, format_spec: &Expr) {
self.enter_node(format_spec);
walk_expr(self, format_spec);
self.exit_node();
}
fn visit_arguments(&mut self, arguments: &Arguments) {
self.enter_node(arguments);
walk_arguments(self, arguments);
self.exit_node();
}
fn visit_arg(&mut self, arg: &Arg) {
self.enter_node(arg);
walk_arg(self, arg);
self.exit_node();
}
fn visit_keyword(&mut self, keyword: &Keyword) {
self.enter_node(keyword);
walk_keyword(self, keyword);
self.exit_node();
}
fn visit_alias(&mut self, alias: &Alias) {
self.enter_node(alias);
walk_alias(self, alias);
self.exit_node();
}
fn visit_with_item(&mut self, with_item: &WithItem) {
self.enter_node(with_item);
walk_with_item(self, with_item);
self.exit_node();
}
fn visit_match_case(&mut self, match_case: &MatchCase) {
self.enter_node(match_case);
walk_match_case(self, match_case);
self.exit_node();
}
fn visit_pattern(&mut self, pattern: &Pattern) {
self.enter_node(pattern);
walk_pattern(self, pattern);
self.exit_node();
}
fn visit_type_param(&mut self, type_param: &TypeParam) {
self.enter_node(type_param);
walk_type_param(self, type_param);
self.exit_node();
}
}
}

291
crates/ruff_python_ast/src/visitor/preorder.rs
View file

@ -1,4 +1,4 @@
use rustpython_ast::{
use crate::{
self as ast, Alias, Arg, ArgWithDefault, Arguments, BoolOp, CmpOp, Comprehension, Constant,
Decorator, ElifElseClause, ExceptHandler, Expr, Keyword, MatchCase, Mod, Operator, Pattern,
Stmt, TypeIgnore, TypeParam, TypeParamTypeVar, UnaryOp, WithItem,
@ -981,292 +981,3 @@ where
V: PreorderVisitor<'a> + ?Sized,
{
}
#[cfg(test)]
mod tests {
use std::fmt::{Debug, Write};
use insta::assert_snapshot;
use rustpython_parser::lexer::lex;
use rustpython_parser::{parse_tokens, Mode};
use crate::node::AnyNodeRef;
use crate::visitor::preorder::{
walk_alias, walk_arg, walk_arguments, walk_comprehension, walk_except_handler, walk_expr,
walk_keyword, walk_match_case, walk_module, walk_pattern, walk_stmt, walk_type_ignore,
walk_type_param, walk_with_item, Alias, Arg, Arguments, BoolOp, CmpOp, Comprehension,
Constant, ExceptHandler, Expr, Keyword, MatchCase, Mod, Operator, Pattern, PreorderVisitor,
Stmt, TypeIgnore, TypeParam, UnaryOp, WithItem,
};
#[test]
fn function_arguments() {
let source = r#"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_positional_only_with_default() {
let source = r#"def a(b, c = 34,/, e = 20, *args): pass"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn compare() {
let source = r#"4 < x < 5"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn list_comprehension() {
let source = "[x for x in numbers]";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn dict_comprehension() {
let source = "{x: x**2 for x in numbers}";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn set_comprehension() {
let source = "{x for x in numbers}";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn match_class_pattern() {
let source = r#"
match x:
case Point2D(0, 0):
...
case Point3D(x=0, y=0, z=0):
...
"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn decorators() {
let source = r#"
@decorator
def a():
pass
@test
class A:
pass
"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn type_aliases() {
let source = r#"type X[T: str, U, *Ts, **P] = list[T]"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn class_type_parameters() {
let source = r#"class X[T: str, U, *Ts, **P]: ..."#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_type_parameters() {
let source = r#"def X[T: str, U, *Ts, **P](): ..."#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
fn trace_preorder_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, Mode::Module, "test.py").unwrap();
let mut visitor = RecordVisitor::default();
visitor.visit_mod(&parsed);
visitor.output
}
/// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
/// and leafs for attributes.
#[derive(Default)]
struct RecordVisitor {
depth: usize,
output: String,
}
impl RecordVisitor {
fn enter_node<'a, T>(&mut self, node: T)
where
T: Into<AnyNodeRef<'a>>,
{
self.emit(&node.into().kind());
self.depth += 1;
}
fn exit_node(&mut self) {
self.depth -= 1;
}
fn emit(&mut self, text: &dyn Debug) {
for _ in 0..self.depth {
self.output.push_str(" ");
}
writeln!(self.output, "- {text:?}").unwrap();
}
}
impl PreorderVisitor<'_> for RecordVisitor {
fn visit_mod(&mut self, module: &Mod) {
self.enter_node(module);
walk_module(self, module);
self.exit_node();
}
fn visit_stmt(&mut self, stmt: &Stmt) {
self.enter_node(stmt);
walk_stmt(self, stmt);
self.exit_node();
}
fn visit_annotation(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_expr(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_constant(&mut self, constant: &Constant) {
self.emit(&constant);
}
fn visit_bool_op(&mut self, bool_op: &BoolOp) {
self.emit(&bool_op);
}
fn visit_operator(&mut self, operator: &Operator) {
self.emit(&operator);
}
fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
self.emit(&unary_op);
}
fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
self.emit(&cmp_op);
}
fn visit_comprehension(&mut self, comprehension: &Comprehension) {
self.enter_node(comprehension);
walk_comprehension(self, comprehension);
self.exit_node();
}
fn visit_except_handler(&mut self, except_handler: &ExceptHandler) {
self.enter_node(except_handler);
walk_except_handler(self, except_handler);
self.exit_node();
}
fn visit_format_spec(&mut self, format_spec: &Expr) {
self.enter_node(format_spec);
walk_expr(self, format_spec);
self.exit_node();
}
fn visit_arguments(&mut self, arguments: &Arguments) {
self.enter_node(arguments);
walk_arguments(self, arguments);
self.exit_node();
}
fn visit_arg(&mut self, arg: &Arg) {
self.enter_node(arg);
walk_arg(self, arg);
self.exit_node();
}
fn visit_keyword(&mut self, keyword: &Keyword) {
self.enter_node(keyword);
walk_keyword(self, keyword);
self.exit_node();
}
fn visit_alias(&mut self, alias: &Alias) {
self.enter_node(alias);
walk_alias(self, alias);
self.exit_node();
}
fn visit_with_item(&mut self, with_item: &WithItem) {
self.enter_node(with_item);
walk_with_item(self, with_item);
self.exit_node();
}
fn visit_match_case(&mut self, match_case: &MatchCase) {
self.enter_node(match_case);
walk_match_case(self, match_case);
self.exit_node();
}
fn visit_pattern(&mut self, pattern: &Pattern) {
self.enter_node(pattern);
walk_pattern(self, pattern);
self.exit_node();
}
fn visit_type_ignore(&mut self, type_ignore: &TypeIgnore) {
self.enter_node(type_ignore);
walk_type_ignore(self, type_ignore);
self.exit_node();
}
fn visit_type_param(&mut self, type_param: &TypeParam) {
self.enter_node(type_param);
walk_type_param(self, type_param);
self.exit_node();
}
}
}

15
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__class_type_parameters.snap
View file

@ -1,15 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtClassDef
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- StmtExpr
- ExprConstant
- Ellipsis

15
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__compare.snap
View file

@ -1,15 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtExpr
- ExprCompare
- ExprConstant
- Int(4)
- Lt
- ExprName
- Lt
- ExprConstant
- Int(5)

13
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__decorators.snap
View file

@ -1,13 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtFunctionDef
- ExprName
- Arguments
- StmtPass
- StmtClassDef
- ExprName
- StmtPass

17
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__dict_comprehension.snap
View file

@ -1,17 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtExpr
- ExprDictComp
- ExprName
- ExprBinOp
- ExprName
- Pow
- ExprConstant
- Int(2)
- Comprehension
- ExprName
- ExprName

23
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__function_arguments.snap
View file

@ -1,23 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtFunctionDef
- Arguments
- Arg
- Arg
- Arg
- Arg
- ExprConstant
- Int(20)
- Arg
- Arg
- ExprConstant
- Int(5)
- Arg
- ExprConstant
- Int(20)
- Arg
- StmtPass

17
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__function_positional_only_with_default.snap
View file

@ -1,17 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtFunctionDef
- Arguments
- Arg
- Arg
- ExprConstant
- Int(34)
- Arg
- ExprConstant
- Int(20)
- Arg
- StmtPass

16
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__function_type_parameters.snap
View file

@ -1,16 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtFunctionDef
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- Arguments
- StmtExpr
- ExprConstant
- Ellipsis

12
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__list_comprehension.snap
View file

@ -1,12 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtExpr
- ExprListComp
- ExprName
- Comprehension
- ExprName
- ExprName

35
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__match_class_pattern.snap
View file

@ -1,35 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtMatch
- ExprName
- MatchCase
- PatternMatchClass
- ExprName
- PatternMatchValue
- ExprConstant
- Int(0)
- PatternMatchValue
- ExprConstant
- Int(0)
- StmtExpr
- ExprConstant
- Ellipsis
- MatchCase
- PatternMatchClass
- ExprName
- PatternMatchValue
- ExprConstant
- Int(0)
- PatternMatchValue
- ExprConstant
- Int(0)
- PatternMatchValue
- ExprConstant
- Int(0)
- StmtExpr
- ExprConstant
- Ellipsis

12
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__set_comprehension.snap
View file

@ -1,12 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtExpr
- ExprSetComp
- ExprName
- Comprehension
- ExprName
- ExprName

16
crates/ruff_python_ast/src/visitor/snapshots/ruff_python_ast__visitor__preorder__tests__type_aliases.snap
View file

@ -1,16 +0,0 @@
---
source: crates/ruff_python_ast/src/visitor/preorder.rs
expression: trace
---
- ModModule
- StmtTypeAlias
- ExprName
- TypeParamTypeVar
- ExprName
- TypeParamTypeVar
- TypeParamTypeVarTuple
- TypeParamParamSpec
- ExprSubscript
- ExprName
- ExprName

2
crates/ruff_python_ast/src/whitespace.rs
View file

@ -1,5 +1,5 @@
use crate::{Ranged, Stmt};
use ruff_text_size::{TextRange, TextSize};
use rustpython_ast::{Ranged, Stmt};
use ruff_python_trivia::{
has_trailing_content, indentation_at_offset, is_python_whitespace, PythonWhitespace,