language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-18 17:41:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Improve internal documentation for the semantic model (#10788)

Browse source
This commit is contained in:
Alex Waygood 2024-04-06 17:28:32 +01:00 • committed by GitHub
parent 7fb5f47efe
commit 1dc93107dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 142 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

21
crates/ruff_linter/src/checkers/ast/analyze/expression.rs
View file

@ -32,10 +32,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if let Some(operator) = typing::to_pep604_operator(value, slice, &checker.semantic)
{
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py310
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
@ -48,7 +47,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py310
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
@ -60,9 +59,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
// Ex) list[...]
if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& typing::is_pep585_generic(value, &checker.semantic)
{
@ -186,10 +184,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
typing::to_pep585_generic(expr, &checker.semantic)
{
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
@ -200,7 +197,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py39
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
@ -270,10 +267,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
]) {
if let Some(replacement) = typing::to_pep585_generic(expr, &checker.semantic) {
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
@ -286,7 +282,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py39
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
@ -1176,9 +1172,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
}) => {
// Ex) `str | None`
if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py310
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
{
flake8_future_annotations::rules::future_required_type_annotation(

9
crates/ruff_linter/src/checkers/ast/annotation.rs
View file

@ -56,9 +56,10 @@ impl AnnotationContext {
_ => {}
}
// If `__future__` annotations are enabled, then annotations are never evaluated
// at runtime, so we can treat them as typing-only.
if semantic.future_annotations() {
// If `__future__` annotations are enabled or it's a stub file,
// then annotations are never evaluated at runtime,
// so we can treat them as typing-only.
if semantic.future_annotations_or_stub() {
return Self::TypingOnly;
}
@ -87,7 +88,7 @@ impl AnnotationContext {
semantic,
) {
Self::RuntimeRequired
} else if semantic.future_annotations() {
} else if semantic.future_annotations_or_stub() {
Self::TypingOnly
} else {
Self::RuntimeEvaluated

73
crates/ruff_linter/src/checkers/ast/mod.rs
View file

@ -935,9 +935,12 @@ impl<'a> Visitor<'a> for Checker<'a> {
fn visit_expr(&mut self, expr: &'a Expr) {
// Step 0: Pre-processing
if !self.semantic.in_typing_literal()
// `in_deferred_type_definition()` will only be `true` if we're now visiting the deferred nodes
// after having already traversed the source tree once. If we're now visiting the deferred nodes,
// we can't defer again, or we'll infinitely recurse!
&& !self.semantic.in_deferred_type_definition()
&& self.semantic.in_type_definition()
&& self.semantic.future_annotations()
&& self.semantic.future_annotations_or_stub()
&& (self.semantic.in_annotation() || self.source_type.is_stub())
{
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = expr {
@ -1964,6 +1967,25 @@ impl<'a> Checker<'a> {
scope.add(id, binding_id);
}
/// After initial traversal of the AST, visit all "future type definitions".
///
/// A "future type definition" is a type definition where [PEP 563] semantics
/// apply (i.e., an annotation in a module that has `from __future__ import annotations`
/// at the top of the file, or an annotation in a stub file). These type definitions
/// support forward references, so they are deferred on initial traversal
/// of the source tree.
///
/// For example:
/// ```python
/// from __future__ import annotations
///
/// def foo() -> Bar: # <-- return annotation is a "future type definition"
/// return Bar()
///
/// class Bar: pass
/// ```
///
/// [PEP 563]: https://peps.python.org/pep-0563/
fn visit_deferred_future_type_definitions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.future_type_definitions.is_empty() {
@ -1971,6 +1993,14 @@ impl<'a> Checker<'a> {
for (expr, snapshot) in type_definitions {
self.semantic.restore(snapshot);
// Type definitions should only be considered "`__future__` type definitions"
// if they are annotations in a module where `from __future__ import
// annotations` is active, or they are type definitions in a stub file.
debug_assert!(
self.semantic.future_annotations_or_stub()
&& (self.source_type.is_stub() || self.semantic.in_annotation())
);
self.semantic.flags |= SemanticModelFlags::TYPE_DEFINITION
| SemanticModelFlags::FUTURE_TYPE_DEFINITION;
self.visit_expr(expr);
@ -1979,6 +2009,19 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}
/// After initial traversal of the AST, visit all [type parameter definitions].
///
/// Type parameters natively support forward references,
/// so are always deferred during initial traversal of the source tree.
///
/// For example:
/// ```python
/// class Foo[T: Bar]: pass # <-- Forward reference used in definition of type parameter `T`
/// type X[T: Bar] = Foo[T] # <-- Ditto
/// class Bar: pass
/// ```
///
/// [type parameter definitions]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes
fn visit_deferred_type_param_definitions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.type_param_definitions.is_empty() {
@ -1994,6 +2037,17 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}
/// After initial traversal of the AST, visit all "string type definitions",
/// i.e., type definitions that are enclosed within quotes so as to allow
/// the type definition to use forward references.
///
/// For example:
/// ```python
/// def foo() -> "Bar": # <-- return annotation is a "string type definition"
/// return Bar()
///
/// class Bar: pass
/// ```
fn visit_deferred_string_type_definitions(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
let snapshot = self.semantic.snapshot();
while !self.visit.string_type_definitions.is_empty() {
@ -2006,7 +2060,7 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
if self.semantic.in_annotation() && self.semantic.future_annotations() {
if self.semantic.in_annotation() && self.semantic.future_annotations_or_stub() {
if self.enabled(Rule::QuotedAnnotation) {
pyupgrade::rules::quoted_annotation(self, value, range);
}
@ -2042,6 +2096,11 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}
/// After initial traversal of the AST, visit all function bodies.
///
/// Function bodies are always deferred on initial traversal of the source tree,
/// as the body of a function may validly contain references to global-scope symbols
/// that were not yet defined at the point when the function was defined.
fn visit_deferred_functions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.functions.is_empty() {
@ -2065,8 +2124,9 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}
/// Visit all deferred lambdas. Returns a list of snapshots, such that the caller can restore
/// the semantic model to the state it was in before visiting the deferred lambdas.
/// After initial traversal of the source tree has been completed,
/// visit all lambdas. Lambdas are deferred during the initial traversal
/// for the same reason as function bodies.
fn visit_deferred_lambdas(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.lambdas.is_empty() {
@ -2092,8 +2152,9 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}
/// Recursively visit all deferred AST nodes, including lambdas, functions, and type
/// annotations.
/// After initial traversal of the source tree has been completed,
/// recursively visit all AST nodes that were deferred on the first pass.
/// This includes lambdas, functions, type parameters, and type annotations.
fn visit_deferred(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
while !self.visit.is_empty() {
self.visit_deferred_functions();

77
crates/ruff_python_semantic/src/model.rs
View file

@ -1465,31 +1465,36 @@ impl<'a> SemanticModel<'a> {
self.flags.intersects(SemanticModelFlags::TYPE_DEFINITION)
}
/// Return `true` if the model is in a string type definition.
/// Return `true` if the model is visiting a "string type definition"
/// that was previously deferred when initially traversing the AST
pub const fn in_string_type_definition(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::STRING_TYPE_DEFINITION)
}
/// Return `true` if the model is in a "simple" string type definition.
/// Return `true` if the model is visiting a "simple string type definition"
/// that was previously deferred when initially traversing the AST
pub const fn in_simple_string_type_definition(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::SIMPLE_STRING_TYPE_DEFINITION)
}
/// Return `true` if the model is in a "complex" string type definition.
/// Return `true` if the model is visiting a "complex string type definition"
/// that was previously deferred when initially traversing the AST
pub const fn in_complex_string_type_definition(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::COMPLEX_STRING_TYPE_DEFINITION)
}
/// Return `true` if the model is in a `__future__` type definition.
/// Return `true` if the model is visiting a "`__future__` type definition"
/// that was previously deferred when initially traversing the AST
pub const fn in_future_type_definition(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::FUTURE_TYPE_DEFINITION)
}
/// Return `true` if the model is in any kind of deferred type definition.
/// Return `true` if the model is visiting any kind of type definition
/// that was previously deferred when initially traversing the AST
pub const fn in_deferred_type_definition(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::DEFERRED_TYPE_DEFINITION)
@ -1574,9 +1579,9 @@ impl<'a> SemanticModel<'a> {
}
/// Return `true` if `__future__`-style type annotations are enabled.
pub const fn future_annotations(&self) -> bool {
pub const fn future_annotations_or_stub(&self) -> bool {
self.flags
.intersects(SemanticModelFlags::FUTURE_ANNOTATIONS)
.intersects(SemanticModelFlags::FUTURE_ANNOTATIONS_OR_STUB)
}
/// Return `true` if the model is in a stub file (i.e., a file with a `.pyi` extension).
@ -1770,6 +1775,9 @@ bitflags! {
///
/// "Simple" string type definitions are those that consist of a single string literal,
/// as opposed to an implicitly concatenated string literal.
///
/// Note that this flag is only set when we are actually *visiting* the deferred definition,
/// not when we "pass by" it when initially traversing the source tree.
const SIMPLE_STRING_TYPE_DEFINITION = 1 << 4;
/// The model is in a (deferred) "complex" string type definition.
@ -1781,6 +1789,9 @@ bitflags! {
///
/// "Complex" string type definitions are those that consist of a implicitly concatenated
/// string literals. These are uncommon but valid.
///
/// Note that this flag is only set when we are actually *visiting* the deferred definition,
/// not when we "pass by" it when initially traversing the source tree.
const COMPLEX_STRING_TYPE_DEFINITION = 1 << 5;
/// The model is in a (deferred) `__future__` type definition.
@ -1794,6 +1805,20 @@ bitflags! {
///
/// `__future__`-style type annotations are only enabled if the `annotations` feature
/// is enabled via `from __future__ import annotations`.
///
/// This flag should only be set in contexts where PEP-563 semantics are relevant to
/// resolution of the type definition. For example, the flag should not be set
/// in the following context, because the type definition is not inside a type annotation,
/// so whether or not `from __future__ import annotations` is active has no relevance:
/// ```python
/// from __future__ import annotations
/// from typing import TypeAlias
///
/// X: TypeAlias = list[int]
/// ```
///
/// Note also that this flag is only set when we are actually *visiting* the deferred definition,
/// not when we "pass by" it when initially traversing the source tree.
const FUTURE_TYPE_DEFINITION = 1 << 6;
/// The model is in an exception handler.
@ -1884,7 +1909,8 @@ bitflags! {
/// any other non-`__future__`-importing statements.
const FUTURES_BOUNDARY = 1 << 14;
/// `__future__`-style type annotations are enabled in this model.
/// The model is in a file that has `from __future__ import annotations`
/// at the top of the module.
///
/// For example, the model could be visiting `x` in:
/// ```python
@ -1899,6 +1925,12 @@ bitflags! {
/// The model is in a Python stub file (i.e., a `.pyi` file).
const STUB_FILE = 1 << 16;
/// `__future__`-style type annotations are enabled in this model.
/// That could be because it's a stub file,
/// or it could be because it's a non-stub file that has `from __future__ import annotations`
/// a the top of the module.
const FUTURE_ANNOTATIONS_OR_STUB = Self::FUTURE_ANNOTATIONS.bits() | Self::STUB_FILE.bits();
/// The model has traversed past the module docstring.
///
/// For example, the model could be visiting `x` in:
@ -1909,14 +1941,28 @@ bitflags! {
/// ```
const MODULE_DOCSTRING_BOUNDARY = 1 << 17;
/// The model is in a type parameter definition.
/// The model is in a (deferred) [type parameter definition].
///
/// For example, the model could be visiting `Record` in:
/// For example, the model could be visiting `T`, `P` or `Ts` in:
/// ```python
/// from typing import TypeVar
/// class Foo[T, *Ts, **P]: pass
/// ```
///
/// Record = TypeVar("Record")
/// Note that this flag is *not* set for "pre-PEP-695" TypeVars, ParamSpecs or TypeVarTuples.
/// None of the following would lead to the flag being set:
///
/// ```python
/// from typing import TypeVar, ParamSpec, TypeVarTuple
///
/// T = TypeVar("T")
/// P = ParamSpec("P")
/// Ts = TypeVarTuple("Ts")
/// ```
///
/// Note also that this flag is only set when we are actually *visiting* the deferred definition,
/// not when we "pass by" it when initially traversing the source tree.
///
/// [type parameter definition]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes
const TYPE_PARAM_DEFINITION = 1 << 18;
/// The model is in a named expression assignment.
@ -1996,12 +2042,11 @@ bitflags! {
impl SemanticModelFlags {
pub fn new(path: &Path) -> Self {
let mut flags = Self::default();
if is_python_stub_file(path) {
flags |= Self::STUB_FILE;
flags |= Self::FUTURE_ANNOTATIONS;
Self::STUB_FILE
} else {
Self::default()
}
flags
}
}

1
crates/ruff_python_semantic/src/scope.rs
View file

@ -185,6 +185,7 @@ pub enum ScopeKind<'a> {
Function(&'a ast::StmtFunctionDef),
Generator,
Module,
/// A Python 3.12+ ["annotation scope"](https://docs.python.org/3/reference/executionmodel.html#annotation-scopes)
Type,
Lambda(&'a ast::ExprLambda),
}