diff --git a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs index b3fc11d7f7..e46dad03b7 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs @@ -32,10 +32,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { if let Some(operator) = typing::to_pep604_operator(value, slice, &checker.semantic) { if checker.enabled(Rule::FutureRewritableTypeAnnotation) { - if !checker.source_type.is_stub() + if !checker.semantic.future_annotations_or_stub() && checker.settings.target_version < PythonVersion::Py310 && checker.settings.target_version >= PythonVersion::Py37 - && !checker.semantic.future_annotations() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing { @@ -48,7 +47,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { if checker.source_type.is_stub() || checker.settings.target_version >= PythonVersion::Py310 || (checker.settings.target_version >= PythonVersion::Py37 - && checker.semantic.future_annotations() + && checker.semantic.future_annotations_or_stub() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing) { @@ -60,9 +59,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { // Ex) list[...] if checker.enabled(Rule::FutureRequiredTypeAnnotation) { - if !checker.source_type.is_stub() + if !checker.semantic.future_annotations_or_stub() && checker.settings.target_version < PythonVersion::Py39 - && !checker.semantic.future_annotations() && checker.semantic.in_annotation() && typing::is_pep585_generic(value, &checker.semantic) { @@ -186,10 +184,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { typing::to_pep585_generic(expr, &checker.semantic) { if checker.enabled(Rule::FutureRewritableTypeAnnotation) { - if !checker.source_type.is_stub() + if !checker.semantic.future_annotations_or_stub() && checker.settings.target_version < PythonVersion::Py39 && checker.settings.target_version >= PythonVersion::Py37 - && !checker.semantic.future_annotations() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing { @@ -200,7 +197,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { if checker.source_type.is_stub() || checker.settings.target_version >= PythonVersion::Py39 || (checker.settings.target_version >= PythonVersion::Py37 - && checker.semantic.future_annotations() + && checker.semantic.future_annotations_or_stub() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing) { @@ -270,10 +267,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { ]) { if let Some(replacement) = typing::to_pep585_generic(expr, &checker.semantic) { if checker.enabled(Rule::FutureRewritableTypeAnnotation) { - if !checker.source_type.is_stub() + if !checker.semantic.future_annotations_or_stub() && checker.settings.target_version < PythonVersion::Py39 && checker.settings.target_version >= PythonVersion::Py37 - && !checker.semantic.future_annotations() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing { @@ -286,7 +282,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { if checker.source_type.is_stub() || checker.settings.target_version >= PythonVersion::Py39 || (checker.settings.target_version >= PythonVersion::Py37 - && checker.semantic.future_annotations() + && checker.semantic.future_annotations_or_stub() && checker.semantic.in_annotation() && !checker.settings.pyupgrade.keep_runtime_typing) { @@ -1176,9 +1172,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) { }) => { // Ex) `str | None` if checker.enabled(Rule::FutureRequiredTypeAnnotation) { - if !checker.source_type.is_stub() + if !checker.semantic.future_annotations_or_stub() && checker.settings.target_version < PythonVersion::Py310 - && !checker.semantic.future_annotations() && checker.semantic.in_annotation() { flake8_future_annotations::rules::future_required_type_annotation( diff --git a/crates/ruff_linter/src/checkers/ast/annotation.rs b/crates/ruff_linter/src/checkers/ast/annotation.rs index 1421361a92..86d1ba50f8 100644 --- a/crates/ruff_linter/src/checkers/ast/annotation.rs +++ b/crates/ruff_linter/src/checkers/ast/annotation.rs @@ -56,9 +56,10 @@ impl AnnotationContext { _ => {} } - // If `__future__` annotations are enabled, then annotations are never evaluated - // at runtime, so we can treat them as typing-only. - if semantic.future_annotations() { + // If `__future__` annotations are enabled or it's a stub file, + // then annotations are never evaluated at runtime, + // so we can treat them as typing-only. + if semantic.future_annotations_or_stub() { return Self::TypingOnly; } @@ -87,7 +88,7 @@ impl AnnotationContext { semantic, ) { Self::RuntimeRequired - } else if semantic.future_annotations() { + } else if semantic.future_annotations_or_stub() { Self::TypingOnly } else { Self::RuntimeEvaluated diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 990fa0abef..906df3358b 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -935,9 +935,12 @@ impl<'a> Visitor<'a> for Checker<'a> { fn visit_expr(&mut self, expr: &'a Expr) { // Step 0: Pre-processing if !self.semantic.in_typing_literal() + // `in_deferred_type_definition()` will only be `true` if we're now visiting the deferred nodes + // after having already traversed the source tree once. If we're now visiting the deferred nodes, + // we can't defer again, or we'll infinitely recurse! && !self.semantic.in_deferred_type_definition() && self.semantic.in_type_definition() - && self.semantic.future_annotations() + && self.semantic.future_annotations_or_stub() && (self.semantic.in_annotation() || self.source_type.is_stub()) { if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = expr { @@ -1964,6 +1967,25 @@ impl<'a> Checker<'a> { scope.add(id, binding_id); } + /// After initial traversal of the AST, visit all "future type definitions". + /// + /// A "future type definition" is a type definition where [PEP 563] semantics + /// apply (i.e., an annotation in a module that has `from __future__ import annotations` + /// at the top of the file, or an annotation in a stub file). These type definitions + /// support forward references, so they are deferred on initial traversal + /// of the source tree. + /// + /// For example: + /// ```python + /// from __future__ import annotations + /// + /// def foo() -> Bar: # <-- return annotation is a "future type definition" + /// return Bar() + /// + /// class Bar: pass + /// ``` + /// + /// [PEP 563]: https://peps.python.org/pep-0563/ fn visit_deferred_future_type_definitions(&mut self) { let snapshot = self.semantic.snapshot(); while !self.visit.future_type_definitions.is_empty() { @@ -1971,6 +1993,14 @@ impl<'a> Checker<'a> { for (expr, snapshot) in type_definitions { self.semantic.restore(snapshot); + // Type definitions should only be considered "`__future__` type definitions" + // if they are annotations in a module where `from __future__ import + // annotations` is active, or they are type definitions in a stub file. + debug_assert!( + self.semantic.future_annotations_or_stub() + && (self.source_type.is_stub() || self.semantic.in_annotation()) + ); + self.semantic.flags |= SemanticModelFlags::TYPE_DEFINITION | SemanticModelFlags::FUTURE_TYPE_DEFINITION; self.visit_expr(expr); @@ -1979,6 +2009,19 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); } + /// After initial traversal of the AST, visit all [type parameter definitions]. + /// + /// Type parameters natively support forward references, + /// so are always deferred during initial traversal of the source tree. + /// + /// For example: + /// ```python + /// class Foo[T: Bar]: pass # <-- Forward reference used in definition of type parameter `T` + /// type X[T: Bar] = Foo[T] # <-- Ditto + /// class Bar: pass + /// ``` + /// + /// [type parameter definitions]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes fn visit_deferred_type_param_definitions(&mut self) { let snapshot = self.semantic.snapshot(); while !self.visit.type_param_definitions.is_empty() { @@ -1994,6 +2037,17 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); } + /// After initial traversal of the AST, visit all "string type definitions", + /// i.e., type definitions that are enclosed within quotes so as to allow + /// the type definition to use forward references. + /// + /// For example: + /// ```python + /// def foo() -> "Bar": # <-- return annotation is a "string type definition" + /// return Bar() + /// + /// class Bar: pass + /// ``` fn visit_deferred_string_type_definitions(&mut self, allocator: &'a typed_arena::Arena) { let snapshot = self.semantic.snapshot(); while !self.visit.string_type_definitions.is_empty() { @@ -2006,7 +2060,7 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); - if self.semantic.in_annotation() && self.semantic.future_annotations() { + if self.semantic.in_annotation() && self.semantic.future_annotations_or_stub() { if self.enabled(Rule::QuotedAnnotation) { pyupgrade::rules::quoted_annotation(self, value, range); } @@ -2042,6 +2096,11 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); } + /// After initial traversal of the AST, visit all function bodies. + /// + /// Function bodies are always deferred on initial traversal of the source tree, + /// as the body of a function may validly contain references to global-scope symbols + /// that were not yet defined at the point when the function was defined. fn visit_deferred_functions(&mut self) { let snapshot = self.semantic.snapshot(); while !self.visit.functions.is_empty() { @@ -2065,8 +2124,9 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); } - /// Visit all deferred lambdas. Returns a list of snapshots, such that the caller can restore - /// the semantic model to the state it was in before visiting the deferred lambdas. + /// After initial traversal of the source tree has been completed, + /// visit all lambdas. Lambdas are deferred during the initial traversal + /// for the same reason as function bodies. fn visit_deferred_lambdas(&mut self) { let snapshot = self.semantic.snapshot(); while !self.visit.lambdas.is_empty() { @@ -2092,8 +2152,9 @@ impl<'a> Checker<'a> { self.semantic.restore(snapshot); } - /// Recursively visit all deferred AST nodes, including lambdas, functions, and type - /// annotations. + /// After initial traversal of the source tree has been completed, + /// recursively visit all AST nodes that were deferred on the first pass. + /// This includes lambdas, functions, type parameters, and type annotations. fn visit_deferred(&mut self, allocator: &'a typed_arena::Arena) { while !self.visit.is_empty() { self.visit_deferred_functions(); diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index ac735d97ac..a4be3d1925 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -1465,31 +1465,36 @@ impl<'a> SemanticModel<'a> { self.flags.intersects(SemanticModelFlags::TYPE_DEFINITION) } - /// Return `true` if the model is in a string type definition. + /// Return `true` if the model is visiting a "string type definition" + /// that was previously deferred when initially traversing the AST pub const fn in_string_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::STRING_TYPE_DEFINITION) } - /// Return `true` if the model is in a "simple" string type definition. + /// Return `true` if the model is visiting a "simple string type definition" + /// that was previously deferred when initially traversing the AST pub const fn in_simple_string_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::SIMPLE_STRING_TYPE_DEFINITION) } - /// Return `true` if the model is in a "complex" string type definition. + /// Return `true` if the model is visiting a "complex string type definition" + /// that was previously deferred when initially traversing the AST pub const fn in_complex_string_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::COMPLEX_STRING_TYPE_DEFINITION) } - /// Return `true` if the model is in a `__future__` type definition. + /// Return `true` if the model is visiting a "`__future__` type definition" + /// that was previously deferred when initially traversing the AST pub const fn in_future_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::FUTURE_TYPE_DEFINITION) } - /// Return `true` if the model is in any kind of deferred type definition. + /// Return `true` if the model is visiting any kind of type definition + /// that was previously deferred when initially traversing the AST pub const fn in_deferred_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::DEFERRED_TYPE_DEFINITION) @@ -1574,9 +1579,9 @@ impl<'a> SemanticModel<'a> { } /// Return `true` if `__future__`-style type annotations are enabled. - pub const fn future_annotations(&self) -> bool { + pub const fn future_annotations_or_stub(&self) -> bool { self.flags - .intersects(SemanticModelFlags::FUTURE_ANNOTATIONS) + .intersects(SemanticModelFlags::FUTURE_ANNOTATIONS_OR_STUB) } /// Return `true` if the model is in a stub file (i.e., a file with a `.pyi` extension). @@ -1770,6 +1775,9 @@ bitflags! { /// /// "Simple" string type definitions are those that consist of a single string literal, /// as opposed to an implicitly concatenated string literal. + /// + /// Note that this flag is only set when we are actually *visiting* the deferred definition, + /// not when we "pass by" it when initially traversing the source tree. const SIMPLE_STRING_TYPE_DEFINITION = 1 << 4; /// The model is in a (deferred) "complex" string type definition. @@ -1781,6 +1789,9 @@ bitflags! { /// /// "Complex" string type definitions are those that consist of a implicitly concatenated /// string literals. These are uncommon but valid. + /// + /// Note that this flag is only set when we are actually *visiting* the deferred definition, + /// not when we "pass by" it when initially traversing the source tree. const COMPLEX_STRING_TYPE_DEFINITION = 1 << 5; /// The model is in a (deferred) `__future__` type definition. @@ -1794,6 +1805,20 @@ bitflags! { /// /// `__future__`-style type annotations are only enabled if the `annotations` feature /// is enabled via `from __future__ import annotations`. + /// + /// This flag should only be set in contexts where PEP-563 semantics are relevant to + /// resolution of the type definition. For example, the flag should not be set + /// in the following context, because the type definition is not inside a type annotation, + /// so whether or not `from __future__ import annotations` is active has no relevance: + /// ```python + /// from __future__ import annotations + /// from typing import TypeAlias + /// + /// X: TypeAlias = list[int] + /// ``` + /// + /// Note also that this flag is only set when we are actually *visiting* the deferred definition, + /// not when we "pass by" it when initially traversing the source tree. const FUTURE_TYPE_DEFINITION = 1 << 6; /// The model is in an exception handler. @@ -1884,7 +1909,8 @@ bitflags! { /// any other non-`__future__`-importing statements. const FUTURES_BOUNDARY = 1 << 14; - /// `__future__`-style type annotations are enabled in this model. + /// The model is in a file that has `from __future__ import annotations` + /// at the top of the module. /// /// For example, the model could be visiting `x` in: /// ```python @@ -1899,6 +1925,12 @@ bitflags! { /// The model is in a Python stub file (i.e., a `.pyi` file). const STUB_FILE = 1 << 16; + /// `__future__`-style type annotations are enabled in this model. + /// That could be because it's a stub file, + /// or it could be because it's a non-stub file that has `from __future__ import annotations` + /// a the top of the module. + const FUTURE_ANNOTATIONS_OR_STUB = Self::FUTURE_ANNOTATIONS.bits() | Self::STUB_FILE.bits(); + /// The model has traversed past the module docstring. /// /// For example, the model could be visiting `x` in: @@ -1909,14 +1941,28 @@ bitflags! { /// ``` const MODULE_DOCSTRING_BOUNDARY = 1 << 17; - /// The model is in a type parameter definition. + /// The model is in a (deferred) [type parameter definition]. /// - /// For example, the model could be visiting `Record` in: + /// For example, the model could be visiting `T`, `P` or `Ts` in: /// ```python - /// from typing import TypeVar + /// class Foo[T, *Ts, **P]: pass + /// ``` /// - /// Record = TypeVar("Record") + /// Note that this flag is *not* set for "pre-PEP-695" TypeVars, ParamSpecs or TypeVarTuples. + /// None of the following would lead to the flag being set: /// + /// ```python + /// from typing import TypeVar, ParamSpec, TypeVarTuple + /// + /// T = TypeVar("T") + /// P = ParamSpec("P") + /// Ts = TypeVarTuple("Ts") + /// ``` + /// + /// Note also that this flag is only set when we are actually *visiting* the deferred definition, + /// not when we "pass by" it when initially traversing the source tree. + /// + /// [type parameter definition]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes const TYPE_PARAM_DEFINITION = 1 << 18; /// The model is in a named expression assignment. @@ -1996,12 +2042,11 @@ bitflags! { impl SemanticModelFlags { pub fn new(path: &Path) -> Self { - let mut flags = Self::default(); if is_python_stub_file(path) { - flags |= Self::STUB_FILE; - flags |= Self::FUTURE_ANNOTATIONS; + Self::STUB_FILE + } else { + Self::default() } - flags } } diff --git a/crates/ruff_python_semantic/src/scope.rs b/crates/ruff_python_semantic/src/scope.rs index 0b2637258f..997a6a0252 100644 --- a/crates/ruff_python_semantic/src/scope.rs +++ b/crates/ruff_python_semantic/src/scope.rs @@ -185,6 +185,7 @@ pub enum ScopeKind<'a> { Function(&'a ast::StmtFunctionDef), Generator, Module, + /// A Python 3.12+ ["annotation scope"](https://docs.python.org/3/reference/executionmodel.html#annotation-scopes) Type, Lambda(&'a ast::ExprLambda), }