mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-04 02:38:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Introduce a ruff_python_semantic crate (#3865)

Browse source
This commit is contained in:
Charlie Marsh 2023-04-04 12:50:47 -04:00 • committed by GitHub
parent 46bcb1f725
commit d919adc13c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
64 changed files with 267 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ruff_python_ast/Cargo.toml
View file

@ -8,7 +8,6 @@ rust-version = { workspace = true }
[lib]
[dependencies]
ruff_python_stdlib = { path = "../ruff_python_stdlib" }
ruff_rustpython = { path = "../ruff_rustpython" }
anyhow = { workspace = true }
@ -16,7 +15,6 @@ bitflags = { workspace = true }
is-macro = { workspace = true }
itertools = { workspace = true }
log = { workspace = true }
nohash-hasher = { version = "0.2.0" }
num-bigint = { version = "0.4.3" }
num-traits = { version = "0.2.15" }
once_cell = { workspace = true }
@ -25,4 +23,4 @@ rustc-hash = { workspace = true }
rustpython-common = { workspace = true }
rustpython-parser = { workspace = true }
serde = { workspace = true }
smallvec = { version = "1.10.0" }
smallvec = { workspace = true }

280
crates/ruff_python_ast/src/binding.rs
View file

@ -1,280 +0,0 @@
use std::num::TryFromIntError;
use std::ops::{Deref, Index, IndexMut};
use bitflags::bitflags;
use rustpython_parser::ast::Stmt;
use crate::scope::ScopeId;
use crate::types::{Range, RefEquality};
#[derive(Debug, Clone)]
pub struct Binding<'a> {
pub kind: BindingKind<'a>,
pub range: Range,
/// The context in which the binding was created.
pub context: ExecutionContext,
/// The statement in which the [`Binding`] was defined.
pub source: Option<RefEquality<'a, Stmt>>,
/// Tuple of (scope index, range) indicating the scope and range at which
/// the binding was last used in a runtime context.
pub runtime_usage: Option<(ScopeId, Range)>,
/// Tuple of (scope index, range) indicating the scope and range at which
/// the binding was last used in a typing-time context.
pub typing_usage: Option<(ScopeId, Range)>,
/// Tuple of (scope index, range) indicating the scope and range at which
/// the binding was last used in a synthetic context. This is used for
/// (e.g.) `__future__` imports, explicit re-exports, and other bindings
/// that should be considered used even if they're never referenced.
pub synthetic_usage: Option<(ScopeId, Range)>,
/// The exceptions that were handled when the binding was defined.
pub exceptions: Exceptions,
}
impl<'a> Binding<'a> {
pub fn mark_used(&mut self, scope: ScopeId, range: Range, context: ExecutionContext) {
match context {
ExecutionContext::Runtime => self.runtime_usage = Some((scope, range)),
ExecutionContext::Typing => self.typing_usage = Some((scope, range)),
}
}
pub const fn used(&self) -> bool {
self.runtime_usage.is_some()
|| self.synthetic_usage.is_some()
|| self.typing_usage.is_some()
}
pub const fn is_definition(&self) -> bool {
matches!(
self.kind,
BindingKind::ClassDefinition
| BindingKind::FunctionDefinition
| BindingKind::Builtin
| BindingKind::FutureImportation
| BindingKind::Importation(..)
| BindingKind::FromImportation(..)
| BindingKind::SubmoduleImportation(..)
)
}
pub fn redefines(&self, existing: &'a Binding) -> bool {
match &self.kind {
BindingKind::Importation(Importation { full_name, .. }) => {
if let BindingKind::SubmoduleImportation(SubmoduleImportation {
full_name: existing,
..
}) = &existing.kind
{
return full_name == existing;
}
}
BindingKind::FromImportation(FromImportation { full_name, .. }) => {
if let BindingKind::SubmoduleImportation(SubmoduleImportation {
full_name: existing,
..
}) = &existing.kind
{
return full_name == existing;
}
}
BindingKind::SubmoduleImportation(SubmoduleImportation { full_name, .. }) => {
match &existing.kind {
BindingKind::Importation(Importation {
full_name: existing,
..
})
| BindingKind::SubmoduleImportation(SubmoduleImportation {
full_name: existing,
..
}) => {
return full_name == existing;
}
BindingKind::FromImportation(FromImportation {
full_name: existing,
..
}) => {
return full_name == existing;
}
_ => {}
}
}
BindingKind::Annotation => {
return false;
}
BindingKind::FutureImportation => {
return false;
}
_ => {}
}
existing.is_definition()
}
}
/// ID uniquely identifying a [Binding] in a program.
///
/// Using a `u32` to identify [Binding]s should is sufficient because Ruff only supports documents with a
/// size smaller than or equal to `u32::max`. A document with the size of `u32::max` must have fewer than `u32::max`
/// bindings because bindings must be separated by whitespace (and have an assignment).
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct BindingId(u32);
impl TryFrom<usize> for BindingId {
type Error = TryFromIntError;
fn try_from(value: usize) -> Result<Self, Self::Error> {
Ok(Self(u32::try_from(value)?))
}
}
impl nohash_hasher::IsEnabled for BindingId {}
/// The bindings in a program.
///
/// Bindings are indexed by [`BindingId`]
#[derive(Debug, Clone, Default)]
pub struct Bindings<'a>(Vec<Binding<'a>>);
impl<'a> Bindings<'a> {
/// Pushes a new binding and returns its id
pub fn push(&mut self, binding: Binding<'a>) -> BindingId {
let id = self.next_id();
self.0.push(binding);
id
}
/// Returns the id that will be assigned when pushing the next binding
pub fn next_id(&self) -> BindingId {
BindingId::try_from(self.0.len()).unwrap()
}
}
impl<'a> Index<BindingId> for Bindings<'a> {
type Output = Binding<'a>;
fn index(&self, index: BindingId) -> &Self::Output {
&self.0[usize::from(index)]
}
}
impl<'a> IndexMut<BindingId> for Bindings<'a> {
fn index_mut(&mut self, index: BindingId) -> &mut Self::Output {
&mut self.0[usize::from(index)]
}
}
impl<'a> Deref for Bindings<'a> {
type Target = [Binding<'a>];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<'a> FromIterator<Binding<'a>> for Bindings<'a> {
fn from_iter<T: IntoIterator<Item = Binding<'a>>>(iter: T) -> Self {
Self(Vec::from_iter(iter))
}
}
impl From<BindingId> for usize {
fn from(value: BindingId) -> Self {
value.0 as usize
}
}
#[derive(Debug, Clone)]
pub struct StarImportation<'a> {
/// The level of the import. `None` or `Some(0)` indicate an absolute import.
pub level: Option<usize>,
/// The module being imported. `None` indicates a wildcard import.
pub module: Option<&'a str>,
}
// Pyflakes defines the following binding hierarchy (via inheritance):
// Binding
// ExportBinding
// Annotation
// Argument
// Assignment
// NamedExprAssignment
// Definition
// FunctionDefinition
// ClassDefinition
// Builtin
// Importation
// SubmoduleImportation
// ImportationFrom
// FutureImportation
#[derive(Clone, Debug)]
pub struct Export<'a> {
/// The names of the bindings exported via `__all__`.
pub names: Vec<&'a str>,
}
#[derive(Clone, Debug)]
pub struct Importation<'a> {
/// The name to which the import is bound.
/// Given `import foo`, `name` would be "foo".
/// Given `import foo as bar`, `name` would be "bar".
pub name: &'a str,
/// The full name of the module being imported.
/// Given `import foo`, `full_name` would be "foo".
/// Given `import foo as bar`, `full_name` would be "foo".
pub full_name: &'a str,
}
#[derive(Clone, Debug)]
pub struct FromImportation<'a> {
/// The name to which the import is bound.
/// Given `from foo import bar`, `name` would be "bar".
/// Given `from foo import bar as baz`, `name` would be "baz".
pub name: &'a str,
/// The full name of the module being imported.
/// Given `from foo import bar`, `full_name` would be "foo.bar".
/// Given `from foo import bar as baz`, `full_name` would be "foo.bar".
pub full_name: String,
}
#[derive(Clone, Debug)]
pub struct SubmoduleImportation<'a> {
/// The parent module imported by the submodule import.
/// Given `import foo.bar`, `module` would be "foo".
pub name: &'a str,
/// The full name of the submodule being imported.
/// Given `import foo.bar`, `full_name` would be "foo.bar".
pub full_name: &'a str,
}
#[derive(Clone, Debug, is_macro::Is)]
pub enum BindingKind<'a> {
Annotation,
Argument,
Assignment,
Binding,
LoopVar,
Global,
Nonlocal,
Builtin,
ClassDefinition,
FunctionDefinition,
Export(Export<'a>),
FutureImportation,
Importation(Importation<'a>),
FromImportation(FromImportation<'a>),
SubmoduleImportation(SubmoduleImportation<'a>),
}
bitflags! {
pub struct Exceptions: u32 {
const NAME_ERROR = 0b0000_0001;
const MODULE_NOT_FOUND_ERROR = 0b0000_0010;
const IMPORT_ERROR = 0b0000_0100;
}
}
#[derive(Copy, Debug, Clone)]
pub enum ExecutionContext {
Runtime,
Typing,
}

429
crates/ruff_python_ast/src/context.rs
View file

@ -1,429 +0,0 @@
use std::path::Path;
use nohash_hasher::{BuildNoHashHasher, IntMap};
use rustc_hash::FxHashMap;
use rustpython_parser::ast::{Expr, Stmt};
use smallvec::smallvec;
use ruff_python_stdlib::path::is_python_stub_file;
use ruff_python_stdlib::typing::TYPING_EXTENSIONS;
use crate::binding::{
Binding, BindingId, BindingKind, Bindings, Exceptions, ExecutionContext, FromImportation,
Importation, SubmoduleImportation,
};
use crate::call_path::{collect_call_path, from_unqualified_name, CallPath};
use crate::helpers::from_relative_import;
use crate::scope::{Scope, ScopeId, ScopeKind, ScopeStack, Scopes};
use crate::types::RefEquality;
use crate::typing::AnnotationKind;
use crate::visibility::{module_visibility, Modifier, VisibleScope};
#[allow(clippy::struct_excessive_bools)]
pub struct Context<'a> {
pub typing_modules: &'a [String],
pub module_path: Option<Vec<String>>,
// Retain all scopes and parent nodes, along with a stack of indices to track which are active
// at various points in time.
pub parents: Vec<RefEquality<'a, Stmt>>,
pub depths: FxHashMap<RefEquality<'a, Stmt>, usize>,
pub child_to_parent: FxHashMap<RefEquality<'a, Stmt>, RefEquality<'a, Stmt>>,
// A stack of all bindings created in any scope, at any point in execution.
pub bindings: Bindings<'a>,
// Map from binding index to indexes of bindings that shadow it in other scopes.
pub shadowed_bindings:
std::collections::HashMap<BindingId, Vec<BindingId>, BuildNoHashHasher<BindingId>>,
pub exprs: Vec<RefEquality<'a, Expr>>,
pub scopes: Scopes<'a>,
pub scope_stack: ScopeStack,
pub dead_scopes: Vec<(ScopeId, ScopeStack)>,
// Body iteration; used to peek at siblings.
pub body: &'a [Stmt],
pub body_index: usize,
// Internal, derivative state.
pub visible_scope: VisibleScope,
pub in_annotation: bool,
pub in_type_definition: bool,
pub in_deferred_string_type_definition: Option<AnnotationKind>,
pub in_deferred_type_definition: bool,
pub in_exception_handler: bool,
pub in_f_string: bool,
pub in_literal: bool,
pub in_subscript: bool,
pub in_type_checking_block: bool,
pub seen_import_boundary: bool,
pub futures_allowed: bool,
pub annotations_future_enabled: bool,
pub handled_exceptions: Vec<Exceptions>,
}
impl<'a> Context<'a> {
pub fn new(
typing_modules: &'a [String],
path: &'a Path,
module_path: Option<Vec<String>>,
) -> Self {
let visibility = module_visibility(module_path.as_deref(), path);
Self {
typing_modules,
module_path,
parents: Vec::default(),
depths: FxHashMap::default(),
child_to_parent: FxHashMap::default(),
bindings: Bindings::default(),
shadowed_bindings: IntMap::default(),
exprs: Vec::default(),
scopes: Scopes::default(),
scope_stack: ScopeStack::default(),
dead_scopes: Vec::default(),
body: &[],
body_index: 0,
visible_scope: VisibleScope {
modifier: Modifier::Module,
visibility,
},
in_annotation: false,
in_type_definition: false,
in_deferred_string_type_definition: None,
in_deferred_type_definition: false,
in_exception_handler: false,
in_f_string: false,
in_literal: false,
in_subscript: false,
in_type_checking_block: false,
seen_import_boundary: false,
futures_allowed: true,
annotations_future_enabled: is_python_stub_file(path),
handled_exceptions: Vec::default(),
}
}
/// Return `true` if the `Expr` is a reference to `typing.${target}`.
pub fn match_typing_expr(&self, expr: &Expr, target: &str) -> bool {
self.resolve_call_path(expr).map_or(false, |call_path| {
self.match_typing_call_path(&call_path, target)
})
}
/// Return `true` if the call path is a reference to `typing.${target}`.
pub fn match_typing_call_path(&self, call_path: &CallPath, target: &str) -> bool {
if call_path.as_slice() == ["typing", target] {
return true;
}
if TYPING_EXTENSIONS.contains(target) {
if call_path.as_slice() == ["typing_extensions", target] {
return true;
}
}
if self.typing_modules.iter().any(|module| {
let mut module: CallPath = from_unqualified_name(module);
module.push(target);
*call_path == module
}) {
return true;
}
false
}
/// Return the current `Binding` for a given `name`.
pub fn find_binding(&self, member: &str) -> Option<&Binding> {
self.scopes()
.find_map(|scope| scope.get(member))
.map(|index| &self.bindings[*index])
}
/// Return `true` if `member` is bound as a builtin.
pub fn is_builtin(&self, member: &str) -> bool {
self.find_binding(member)
.map_or(false, |binding| binding.kind.is_builtin())
}
/// Resolves the [`Expr`] to a fully-qualified symbol-name, if `value` resolves to an imported
/// or builtin symbol.
///
/// E.g., given:
///
///
/// ```python
/// from sys import version_info as python_version
/// print(python_version)
/// ```
///
/// ...then `resolve_call_path(${python_version})` will resolve to `sys.version_info`.
pub fn resolve_call_path<'b>(&'a self, value: &'b Expr) -> Option<CallPath<'a>>
where
'b: 'a,
{
let Some(call_path) = collect_call_path(value) else {
return None;
};
let Some(head) = call_path.first() else {
return None;
};
let Some(binding) = self.find_binding(head) else {
return None;
};
match &binding.kind {
BindingKind::Importation(Importation {
full_name: name, ..
})
| BindingKind::SubmoduleImportation(SubmoduleImportation { name, .. }) => {
if name.starts_with('.') {
if let Some(module) = &self.module_path {
let mut source_path = from_relative_import(module, name);
source_path.extend(call_path.into_iter().skip(1));
Some(source_path)
} else {
None
}
} else {
let mut source_path: CallPath = from_unqualified_name(name);
source_path.extend(call_path.into_iter().skip(1));
Some(source_path)
}
}
BindingKind::FromImportation(FromImportation {
full_name: name, ..
}) => {
if name.starts_with('.') {
if let Some(module) = &self.module_path {
let mut source_path = from_relative_import(module, name);
source_path.extend(call_path.into_iter().skip(1));
Some(source_path)
} else {
None
}
} else {
let mut source_path: CallPath = from_unqualified_name(name);
source_path.extend(call_path.into_iter().skip(1));
Some(source_path)
}
}
BindingKind::Builtin => {
let mut source_path: CallPath = smallvec![];
source_path.push("");
source_path.extend(call_path);
Some(source_path)
}
_ => None,
}
}
/// Given a `module` and `member`, return the fully-qualified name of the binding in the current
/// scope, if it exists.
///
/// E.g., given:
///
/// ```python
/// from sys import version_info as python_version
/// print(python_version)
/// ```
///
/// ...then `resolve_qualified_import_name("sys", "version_info")` will return
/// `Some("python_version")`.
pub fn resolve_qualified_import_name(
&self,
module: &str,
member: &str,
) -> Option<(&Stmt, String)> {
self.scopes().enumerate().find_map(|(scope_index, scope)| {
scope.binding_ids().find_map(|binding_index| {
let binding = &self.bindings[*binding_index];
match &binding.kind {
// Ex) Given `module="sys"` and `object="exit"`:
// `import sys` -> `sys.exit`
// `import sys as sys2` -> `sys2.exit`
BindingKind::Importation(Importation { name, full_name }) => {
if full_name == &module {
// Verify that `sys` isn't bound in an inner scope.
if self
.scopes()
.take(scope_index)
.all(|scope| scope.get(name).is_none())
{
return Some((
binding.source.as_ref().unwrap().into(),
format!("{name}.{member}"),
));
}
}
}
// Ex) Given `module="os.path"` and `object="join"`:
// `from os.path import join` -> `join`
// `from os.path import join as join2` -> `join2`
BindingKind::FromImportation(FromImportation { name, full_name }) => {
if let Some((target_module, target_member)) = full_name.split_once('.') {
if target_module == module && target_member == member {
// Verify that `join` isn't bound in an inner scope.
if self
.scopes()
.take(scope_index)
.all(|scope| scope.get(name).is_none())
{
return Some((
binding.source.as_ref().unwrap().into(),
(*name).to_string(),
));
}
}
}
}
// Ex) Given `module="os"` and `object="name"`:
// `import os.path ` -> `os.name`
BindingKind::SubmoduleImportation(SubmoduleImportation { name, .. }) => {
if name == &module {
// Verify that `os` isn't bound in an inner scope.
if self
.scopes()
.take(scope_index)
.all(|scope| scope.get(name).is_none())
{
return Some((
binding.source.as_ref().unwrap().into(),
format!("{name}.{member}"),
));
}
}
}
// Non-imports.
_ => {}
}
None
})
})
}
pub fn push_parent(&mut self, parent: &'a Stmt) {
let num_existing = self.parents.len();
self.parents.push(RefEquality(parent));
self.depths.insert(self.parents[num_existing], num_existing);
if num_existing > 0 {
self.child_to_parent
.insert(self.parents[num_existing], self.parents[num_existing - 1]);
}
}
pub fn pop_parent(&mut self) {
self.parents.pop().expect("Attempted to pop without parent");
}
pub fn push_expr(&mut self, expr: &'a Expr) {
self.exprs.push(RefEquality(expr));
}
pub fn pop_expr(&mut self) {
self.exprs
.pop()
.expect("Attempted to pop without expression");
}
pub fn push_scope(&mut self, kind: ScopeKind<'a>) -> ScopeId {
let id = self.scopes.push_scope(kind);
self.scope_stack.push(id);
id
}
pub fn pop_scope(&mut self) {
self.dead_scopes.push((
self.scope_stack
.pop()
.expect("Attempted to pop without scope"),
self.scope_stack.clone(),
));
}
/// Return the current `Stmt`.
pub fn current_stmt(&self) -> &RefEquality<'a, Stmt> {
self.parents.iter().rev().next().expect("No parent found")
}
/// Return the parent `Stmt` of the current `Stmt`, if any.
pub fn current_stmt_parent(&self) -> Option<&RefEquality<'a, Stmt>> {
self.parents.iter().rev().nth(1)
}
/// Return the parent `Expr` of the current `Expr`.
pub fn current_expr_parent(&self) -> Option<&RefEquality<'a, Expr>> {
self.exprs.iter().rev().nth(1)
}
/// Return the grandparent `Expr` of the current `Expr`.
pub fn current_expr_grandparent(&self) -> Option<&RefEquality<'a, Expr>> {
self.exprs.iter().rev().nth(2)
}
/// Return an [`Iterator`] over the current `Expr` parents.
pub fn expr_ancestors(&self) -> impl Iterator<Item = &RefEquality<'a, Expr>> {
self.exprs.iter().rev().skip(1)
}
/// Return the `Stmt` that immediately follows the current `Stmt`, if any.
pub fn current_sibling_stmt(&self) -> Option<&'a Stmt> {
self.body.get(self.body_index + 1)
}
/// Returns a reference to the global scope
pub fn global_scope(&self) -> &Scope<'a> {
self.scopes.global()
}
/// Returns a mutable reference to the global scope
pub fn global_scope_mut(&mut self) -> &mut Scope<'a> {
self.scopes.global_mut()
}
/// Returns the current top most scope.
pub fn scope(&self) -> &Scope<'a> {
&self.scopes[self.scope_stack.top().expect("No current scope found")]
}
/// Returns the id of the top-most scope
pub fn scope_id(&self) -> ScopeId {
self.scope_stack.top().expect("No current scope found")
}
/// Returns a mutable reference to the current top most scope.
pub fn scope_mut(&mut self) -> &mut Scope<'a> {
let top_id = self.scope_stack.top().expect("No current scope found");
&mut self.scopes[top_id]
}
pub fn parent_scope(&self) -> Option<&Scope> {
self.scope_stack
.iter()
.nth(1)
.map(|index| &self.scopes[*index])
}
pub fn scopes(&self) -> impl Iterator<Item = &Scope> {
self.scope_stack.iter().map(|index| &self.scopes[*index])
}
pub const fn in_exception_handler(&self) -> bool {
self.in_exception_handler
}
/// Return the [`ExecutionContext`] of the current scope.
pub const fn execution_context(&self) -> ExecutionContext {
if self.in_type_checking_block
|| self.in_annotation
|| self.in_deferred_string_type_definition.is_some()
{
ExecutionContext::Typing
} else {
ExecutionContext::Runtime
}
}
/// Return the union of all handled exceptions as an [`Exceptions`] bitflag.
pub fn exceptions(&self) -> Exceptions {
let mut exceptions = Exceptions::empty();
for exception in &self.handled_exceptions {
exceptions.insert(*exception);
}
exceptions
}
}

68
crates/ruff_python_ast/src/function_type.rs
View file

@ -1,68 +0,0 @@
use rustpython_parser::ast::Expr;
use crate::call_path::from_qualified_name;
use crate::context::Context;
use crate::helpers::map_callable;
use crate::scope::{Scope, ScopeKind};
const CLASS_METHODS: [&str; 3] = ["__new__", "__init_subclass__", "__class_getitem__"];
const METACLASS_BASES: [(&str, &str); 2] = [("", "type"), ("abc", "ABCMeta")];
#[derive(Copy, Clone)]
pub enum FunctionType {
Function,
Method,
ClassMethod,
StaticMethod,
}
/// Classify a function based on its scope, name, and decorators.
pub fn classify(
ctx: &Context,
scope: &Scope,
name: &str,
decorator_list: &[Expr],
classmethod_decorators: &[String],
staticmethod_decorators: &[String],
) -> FunctionType {
let ScopeKind::Class(scope) = &scope.kind else {
return FunctionType::Function;
};
if decorator_list.iter().any(|expr| {
// The method is decorated with a static method decorator (like
// `@staticmethod`).
ctx.resolve_call_path(map_callable(expr))
.map_or(false, |call_path| {
call_path.as_slice() == ["", "staticmethod"]
|| staticmethod_decorators
.iter()
.any(|decorator| call_path == from_qualified_name(decorator))
})
}) {
FunctionType::StaticMethod
} else if CLASS_METHODS.contains(&name)
// Special-case class method, like `__new__`.
|| scope.bases.iter().any(|expr| {
// The class itself extends a known metaclass, so all methods are class methods.
ctx.resolve_call_path(map_callable(expr)).map_or(false, |call_path| {
METACLASS_BASES
.iter()
.any(|(module, member)| call_path.as_slice() == [*module, *member])
})
})
|| decorator_list.iter().any(|expr| {
// The method is decorated with a class method decorator (like `@classmethod`).
ctx.resolve_call_path(map_callable(expr)).map_or(false, |call_path| {
call_path.as_slice() == ["", "classmethod"] ||
classmethod_decorators
.iter()
.any(|decorator| call_path == from_qualified_name(decorator))
})
})
{
FunctionType::ClassMethod
} else {
// It's an instance method.
FunctionType::Method
}
}

9
crates/ruff_python_ast/src/helpers.rs
View file

@ -13,7 +13,6 @@ use rustpython_parser::{lexer, Mode, Tok};
use smallvec::SmallVec;
use crate::call_path::CallPath;
use crate::context::Context;
use crate::source_code::{Generator, Indexer, Locator, Stylist};
use crate::types::Range;
use crate::visitor;
@ -50,14 +49,6 @@ pub fn unparse_constant(constant: &Constant, stylist: &Stylist) -> String {
generator.generate()
}
/// Return `true` if the `Expr` contains a reference to `${module}.${target}`.
pub fn contains_call_path(ctx: &Context, expr: &Expr, target: &[&str]) -> bool {
any_over_expr(expr, &|expr| {
ctx.resolve_call_path(expr)
.map_or(false, |call_path| call_path.as_slice() == target)
})
}
/// Return `true` if the `Expr` contains an expression that appears to include a
/// side-effect (like a function call).
///

6
crates/ruff_python_ast/src/lib.rs
View file

@ -1,23 +1,17 @@
pub mod all;
pub mod binding;
pub mod branch_detection;
pub mod call_path;
pub mod cast;
pub mod comparable;
pub mod context;
pub mod function_type;
pub mod hashable;
pub mod helpers;
pub mod imports;
pub mod logging;
pub mod newlines;
pub mod relocate;
pub mod scope;
pub mod source_code;
pub mod str;
pub mod token_kind;
pub mod types;
pub mod typing;
pub mod visibility;
pub mod visitor;
pub mod whitespace;

58
crates/ruff_python_ast/src/logging.rs
View file

@ -1,58 +0,0 @@
use rustpython_parser::ast::{Expr, ExprKind};
use crate::call_path::collect_call_path;
use crate::context::Context;
#[derive(Copy, Clone)]
pub enum LoggingLevel {
Debug,
Critical,
Error,
Exception,
Info,
Warn,
Warning,
}
impl LoggingLevel {
pub fn from_attribute(level: &str) -> Option<Self> {
match level {
"debug" => Some(LoggingLevel::Debug),
"critical" => Some(LoggingLevel::Critical),
"error" => Some(LoggingLevel::Error),
"exception" => Some(LoggingLevel::Exception),
"info" => Some(LoggingLevel::Info),
"warn" => Some(LoggingLevel::Warn),
"warning" => Some(LoggingLevel::Warning),
_ => None,
}
}
}
/// Return `true` if the given `Expr` is a potential logging call. Matches
/// `logging.error`, `logger.error`, `self.logger.error`, etc., but not
/// arbitrary `foo.error` calls.
///
/// It even matches direct `logging.error` calls even if the `logging` module
/// is aliased. Example:
/// ```python
/// import logging as bar
///
/// # This is detected to be a logger candidate
/// bar.error()
/// ```
pub fn is_logger_candidate(context: &Context, func: &Expr) -> bool {
if let ExprKind::Attribute { value, .. } = &func.node {
let Some(call_path) = context
.resolve_call_path(value)
.or_else(|| collect_call_path(value)) else {
return false;
};
if let Some(tail) = call_path.last() {
if tail.starts_with("log") || tail.ends_with("logger") || tail.ends_with("logging") {
return true;
}
}
}
false
}

266
crates/ruff_python_ast/src/scope.rs
View file

@ -1,266 +0,0 @@
use std::num::TryFromIntError;
use std::ops::{Deref, Index, IndexMut};
use rustc_hash::FxHashMap;
use rustpython_parser::ast::{Arguments, Expr, Keyword, Stmt};
use crate::binding::{BindingId, StarImportation};
#[derive(Debug)]
pub struct Scope<'a> {
pub id: ScopeId,
pub kind: ScopeKind<'a>,
pub uses_locals: bool,
/// A list of star imports in this scope. These represent _module_ imports (e.g., `sys` in
/// `from sys import *`), rather than individual bindings (e.g., individual members in `sys`).
star_imports: Vec<StarImportation<'a>>,
/// A map from bound name to binding index, for current bindings.
bindings: FxHashMap<&'a str, BindingId>,
/// A map from bound name to binding index, for bindings that were shadowed later in the scope.
shadowed_bindings: FxHashMap<&'a str, Vec<BindingId>>,
}
impl<'a> Scope<'a> {
pub fn global() -> Self {
Scope::local(ScopeId::global(), ScopeKind::Module)
}
pub fn local(id: ScopeId, kind: ScopeKind<'a>) -> Self {
Scope {
id,
kind,
uses_locals: false,
star_imports: Vec::default(),
bindings: FxHashMap::default(),
shadowed_bindings: FxHashMap::default(),
}
}
/// Returns the [id](BindingId) of the binding bound to the given name.
pub fn get(&self, name: &str) -> Option<&BindingId> {
self.bindings.get(name)
}
/// Adds a new binding with the given name to this scope.
pub fn add(&mut self, name: &'a str, id: BindingId) -> Option<BindingId> {
if let Some(id) = self.bindings.insert(name, id) {
self.shadowed_bindings.entry(name).or_default().push(id);
Some(id)
} else {
None
}
}
/// Returns `true` if this scope defines a binding with the given name.
pub fn defines(&self, name: &str) -> bool {
self.bindings.contains_key(name)
}
/// Removes the binding with the given name
pub fn remove(&mut self, name: &str) -> Option<BindingId> {
self.bindings.remove(name)
}
/// Returns the ids of all bindings defined in this scope.
pub fn binding_ids(&self) -> std::collections::hash_map::Values<&str, BindingId> {
self.bindings.values()
}
/// Returns a tuple of the name and id of all bindings defined in this scope.
pub fn bindings(&self) -> std::collections::hash_map::Iter<&'a str, BindingId> {
self.bindings.iter()
}
/// Returns an iterator over all [bindings](BindingId) bound to the given name, including
/// those that were shadowed by later bindings.
pub fn bindings_for_name(&self, name: &str) -> impl Iterator<Item = &BindingId> {
self.bindings
.get(name)
.into_iter()
.chain(self.shadowed_bindings.get(name).into_iter().flatten().rev())
}
/// Adds a reference to a star import (e.g., `from sys import *`) to this scope.
pub fn add_star_import(&mut self, import: StarImportation<'a>) {
self.star_imports.push(import);
}
/// Returns `true` if this scope contains a star import (e.g., `from sys import *`).
pub fn uses_star_imports(&self) -> bool {
!self.star_imports.is_empty()
}
/// Returns an iterator over all star imports (e.g., `from sys import *`) in this scope.
pub fn star_imports(&self) -> impl Iterator<Item = &StarImportation<'a>> {
self.star_imports.iter()
}
}
/// Id uniquely identifying a scope in a program.
///
/// Using a `u32` is sufficient because Ruff only supports parsing documents with a size of max `u32::max`
/// and it is impossible to have more scopes than characters in the file (because defining a function or class
/// requires more than one character).
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)]
pub struct ScopeId(u32);
impl ScopeId {
/// Returns the ID for the global scope
#[inline]
pub const fn global() -> Self {
ScopeId(0)
}
/// Returns `true` if this is the id of the global scope
pub const fn is_global(&self) -> bool {
self.0 == 0
}
}
impl TryFrom<usize> for ScopeId {
type Error = TryFromIntError;
fn try_from(value: usize) -> Result<Self, Self::Error> {
Ok(Self(u32::try_from(value)?))
}
}
impl From<ScopeId> for usize {
fn from(value: ScopeId) -> Self {
value.0 as usize
}
}
#[derive(Debug)]
pub enum ScopeKind<'a> {
Class(ClassDef<'a>),
Function(FunctionDef<'a>),
Generator,
Module,
Lambda(Lambda<'a>),
}
#[derive(Debug)]
pub struct FunctionDef<'a> {
// Properties derived from StmtKind::FunctionDef.
pub name: &'a str,
pub args: &'a Arguments,
pub body: &'a [Stmt],
pub decorator_list: &'a [Expr],
// pub returns: Option<&'a Expr>,
// pub type_comment: Option<&'a str>,
// Scope-specific properties.
// TODO(charlie): Create AsyncFunctionDef to mirror the AST.
pub async_: bool,
pub globals: FxHashMap<&'a str, &'a Stmt>,
}
#[derive(Debug)]
pub struct ClassDef<'a> {
// Properties derived from StmtKind::ClassDef.
pub name: &'a str,
pub bases: &'a [Expr],
pub keywords: &'a [Keyword],
// pub body: &'a [Stmt],
pub decorator_list: &'a [Expr],
// Scope-specific properties.
pub globals: FxHashMap<&'a str, &'a Stmt>,
}
#[derive(Debug)]
pub struct Lambda<'a> {
pub args: &'a Arguments,
pub body: &'a Expr,
}
/// The scopes of a program indexed by [`ScopeId`]
#[derive(Debug)]
pub struct Scopes<'a>(Vec<Scope<'a>>);
impl<'a> Scopes<'a> {
/// Returns a reference to the global scope
pub fn global(&self) -> &Scope<'a> {
&self[ScopeId::global()]
}
/// Returns a mutable reference to the global scope
pub fn global_mut(&mut self) -> &mut Scope<'a> {
&mut self[ScopeId::global()]
}
/// Pushes a new scope and returns its unique id
pub(crate) fn push_scope(&mut self, kind: ScopeKind<'a>) -> ScopeId {
let next_id = ScopeId::try_from(self.0.len()).unwrap();
self.0.push(Scope::local(next_id, kind));
next_id
}
}
impl Default for Scopes<'_> {
fn default() -> Self {
Self(vec![Scope::global()])
}
}
impl<'a> Index<ScopeId> for Scopes<'a> {
type Output = Scope<'a>;
fn index(&self, index: ScopeId) -> &Self::Output {
&self.0[usize::from(index)]
}
}
impl<'a> IndexMut<ScopeId> for Scopes<'a> {
fn index_mut(&mut self, index: ScopeId) -> &mut Self::Output {
&mut self.0[usize::from(index)]
}
}
impl<'a> Deref for Scopes<'a> {
type Target = [Scope<'a>];
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub struct ScopeStack(Vec<ScopeId>);
impl ScopeStack {
/// Pushes a new scope on the stack
pub fn push(&mut self, id: ScopeId) {
self.0.push(id);
}
/// Pops the top most scope
pub fn pop(&mut self) -> Option<ScopeId> {
self.0.pop()
}
/// Returns the id of the top-most
pub fn top(&self) -> Option<ScopeId> {
self.0.last().copied()
}
/// Returns an iterator from the current scope to the top scope (reverse iterator)
pub fn iter(&self) -> std::iter::Rev<std::slice::Iter<ScopeId>> {
self.0.iter().rev()
}
pub fn snapshot(&self) -> ScopeStackSnapshot {
ScopeStackSnapshot(self.0.len())
}
#[allow(clippy::needless_pass_by_value)]
pub fn restore(&mut self, snapshot: ScopeStackSnapshot) {
self.0.truncate(snapshot.0);
}
}
pub struct ScopeStackSnapshot(usize);
impl Default for ScopeStack {
fn default() -> Self {
Self(vec![ScopeId::global()])
}
}

70
crates/ruff_python_ast/src/typing.rs
View file

@ -1,80 +1,12 @@
use anyhow::Result;
use rustpython_parser as parser;
use rustpython_parser::ast::{Expr, ExprKind, Location};
use rustpython_parser::ast::{Expr, Location};
use ruff_python_stdlib::typing::{PEP_585_BUILTINS_ELIGIBLE, PEP_593_SUBSCRIPTS, SUBSCRIPTS};
use crate::call_path::{from_unqualified_name, CallPath};
use crate::context::Context;
use crate::relocate::relocate_expr;
use crate::source_code::Locator;
use crate::str;
use crate::types::Range;
#[derive(Copy, Clone)]
pub enum Callable {
Cast,
NewType,
TypeVar,
NamedTuple,
TypedDict,
MypyExtension,
}
#[derive(Copy, Clone)]
pub enum SubscriptKind {
AnnotatedSubscript,
PEP593AnnotatedSubscript,
}
pub fn match_annotated_subscript<'a>(
expr: &Expr,
context: &Context,
typing_modules: impl Iterator<Item = &'a str>,
) -> Option<SubscriptKind> {
if !matches!(
expr.node,
ExprKind::Name { .. } | ExprKind::Attribute { .. }
) {
return None;
}
context.resolve_call_path(expr).and_then(|call_path| {
if SUBSCRIPTS.contains(&call_path.as_slice()) {
return Some(SubscriptKind::AnnotatedSubscript);
}
if PEP_593_SUBSCRIPTS.contains(&call_path.as_slice()) {
return Some(SubscriptKind::PEP593AnnotatedSubscript);
}
for module in typing_modules {
let module_call_path: CallPath = from_unqualified_name(module);
if call_path.starts_with(&module_call_path) {
for subscript in SUBSCRIPTS.iter() {
if call_path.last() == subscript.last() {
return Some(SubscriptKind::AnnotatedSubscript);
}
}
for subscript in PEP_593_SUBSCRIPTS.iter() {
if call_path.last() == subscript.last() {
return Some(SubscriptKind::PEP593AnnotatedSubscript);
}
}
}
}
None
})
}
/// Returns `true` if `Expr` represents a reference to a typing object with a
/// PEP 585 built-in.
pub fn is_pep585_builtin(expr: &Expr, context: &Context) -> bool {
context.resolve_call_path(expr).map_or(false, |call_path| {
PEP_585_BUILTINS_ELIGIBLE.contains(&call_path.as_slice())
})
}
#[derive(is_macro::Is, Copy, Clone)]
pub enum AnnotationKind {
/// The annotation is defined as part a simple string literal,

217
crates/ruff_python_ast/src/visibility.rs
View file

@ -1,217 +0,0 @@
use std::path::Path;
use rustpython_parser::ast::{Expr, Stmt, StmtKind};
use crate::call_path::collect_call_path;
use crate::call_path::CallPath;
use crate::context::Context;
use crate::helpers::map_callable;
#[derive(Debug, Clone, Copy)]
pub enum Modifier {
Module,
Class,
Function,
}
#[derive(Debug, Clone, Copy)]
pub enum Visibility {
Public,
Private,
}
#[derive(Debug, Clone, Copy)]
pub struct VisibleScope {
pub modifier: Modifier,
pub visibility: Visibility,
}
/// Returns `true` if a function is a "static method".
pub fn is_staticmethod(ctx: &Context, decorator_list: &[Expr]) -> bool {
decorator_list.iter().any(|expr| {
ctx.resolve_call_path(map_callable(expr))
.map_or(false, |call_path| {
call_path.as_slice() == ["", "staticmethod"]
})
})
}
/// Returns `true` if a function is a "class method".
pub fn is_classmethod(ctx: &Context, decorator_list: &[Expr]) -> bool {
decorator_list.iter().any(|expr| {
ctx.resolve_call_path(map_callable(expr))
.map_or(false, |call_path| {
call_path.as_slice() == ["", "classmethod"]
})
})
}
/// Returns `true` if a function definition is an `@overload`.
pub fn is_overload(ctx: &Context, decorator_list: &[Expr]) -> bool {
decorator_list
.iter()
.any(|expr| ctx.match_typing_expr(map_callable(expr), "overload"))
}
/// Returns `true` if a function definition is an `@override` (PEP 698).
pub fn is_override(ctx: &Context, decorator_list: &[Expr]) -> bool {
decorator_list
.iter()
.any(|expr| ctx.match_typing_expr(map_callable(expr), "override"))
}
/// Returns `true` if a function definition is an `@abstractmethod`.
pub fn is_abstract(ctx: &Context, decorator_list: &[Expr]) -> bool {
decorator_list.iter().any(|expr| {
ctx.resolve_call_path(map_callable(expr))
.map_or(false, |call_path| {
call_path.as_slice() == ["abc", "abstractmethod"]
|| call_path.as_slice() == ["abc", "abstractproperty"]
})
})
}
/// Returns `true` if a function definition is a `@property`.
/// `extra_properties` can be used to check additional non-standard
/// `@property`-like decorators.
pub fn is_property(ctx: &Context, decorator_list: &[Expr], extra_properties: &[CallPath]) -> bool {
decorator_list.iter().any(|expr| {
ctx.resolve_call_path(map_callable(expr))
.map_or(false, |call_path| {
call_path.as_slice() == ["", "property"]
|| call_path.as_slice() == ["functools", "cached_property"]
|| extra_properties
.iter()
.any(|extra_property| extra_property.as_slice() == call_path.as_slice())
})
})
}
/// Returns `true` if a function is a "magic method".
pub fn is_magic(name: &str) -> bool {
name.starts_with("__") && name.ends_with("__")
}
/// Returns `true` if a function is an `__init__`.
pub fn is_init(name: &str) -> bool {
name == "__init__"
}
/// Returns `true` if a function is a `__new__`.
pub fn is_new(name: &str) -> bool {
name == "__new__"
}
/// Returns `true` if a function is a `__call__`.
pub fn is_call(name: &str) -> bool {
name == "__call__"
}
/// Returns `true` if a function is a test one.
pub fn is_test(name: &str) -> bool {
name == "runTest" || name.starts_with("test")
}
/// Returns `true` if a module name indicates public visibility.
fn is_public_module(module_name: &str) -> bool {
!module_name.starts_with('_') || (module_name.starts_with("__") && module_name.ends_with("__"))
}
/// Returns `true` if a module name indicates private visibility.
fn is_private_module(module_name: &str) -> bool {
!is_public_module(module_name)
}
/// Return the stem of a module name (everything preceding the last dot).
fn stem(path: &str) -> &str {
if let Some(index) = path.rfind('.') {
&path[..index]
} else {
path
}
}
/// Return the `Visibility` of the Python file at `Path` based on its name.
pub fn module_visibility(module_path: Option<&[String]>, path: &Path) -> Visibility {
if let Some(module_path) = module_path {
if module_path.iter().any(|m| is_private_module(m)) {
return Visibility::Private;
}
} else {
// When module_path is None, path is a script outside a package, so just
// check to see if the module name itself is private.
// Ex) `_foo.py` (but not `__init__.py`)
let mut components = path.iter().rev();
if let Some(filename) = components.next() {
let module_name = filename.to_string_lossy();
let module_name = stem(&module_name);
if is_private_module(module_name) {
return Visibility::Private;
}
}
}
Visibility::Public
}
pub fn function_visibility(stmt: &Stmt) -> Visibility {
match &stmt.node {
StmtKind::FunctionDef { name, .. } | StmtKind::AsyncFunctionDef { name, .. } => {
if name.starts_with('_') {
Visibility::Private
} else {
Visibility::Public
}
}
_ => panic!("Found non-FunctionDef in function_visibility"),
}
}
pub fn method_visibility(stmt: &Stmt) -> Visibility {
match &stmt.node {
StmtKind::FunctionDef {
name,
decorator_list,
..
}
| StmtKind::AsyncFunctionDef {
name,
decorator_list,
..
} => {
// Is this a setter or deleter?
if decorator_list.iter().any(|expr| {
collect_call_path(expr).map_or(false, |call_path| {
call_path.as_slice() == [name, "setter"]
|| call_path.as_slice() == [name, "deleter"]
})
}) {
return Visibility::Private;
}
// Is the method non-private?
if !name.starts_with('_') {
return Visibility::Public;
}
// Is this a magic method?
if name.starts_with("__") && name.ends_with("__") {
return Visibility::Public;
}
Visibility::Private
}
_ => panic!("Found non-FunctionDef in method_visibility"),
}
}
pub fn class_visibility(stmt: &Stmt) -> Visibility {
match &stmt.node {
StmtKind::ClassDef { name, .. } => {
if name.starts_with('_') {
Visibility::Private
} else {
Visibility::Public
}
}
_ => panic!("Found non-ClassDef in function_visibility"),
}
}