Improve internal documentation for the semantic model (#10788)

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -32,10 +32,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                 if let Some(operator) = typing::to_pep604_operator(value, slice, &checker.semantic)
                 {
                     if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
-                        if !checker.source_type.is_stub()
+                        if !checker.semantic.future_annotations_or_stub()
                             && checker.settings.target_version < PythonVersion::Py310
                             && checker.settings.target_version >= PythonVersion::Py37
-                            && !checker.semantic.future_annotations()
                             && checker.semantic.in_annotation()
                             && !checker.settings.pyupgrade.keep_runtime_typing
                         {
@@ -48,7 +47,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                         if checker.source_type.is_stub()
                             || checker.settings.target_version >= PythonVersion::Py310
                             || (checker.settings.target_version >= PythonVersion::Py37
-                                && checker.semantic.future_annotations()
+                                && checker.semantic.future_annotations_or_stub()
                                 && checker.semantic.in_annotation()
                                 && !checker.settings.pyupgrade.keep_runtime_typing)
                         {
@@ -60,9 +59,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
 
             // Ex) list[...]
             if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
-                if !checker.source_type.is_stub()
+                if !checker.semantic.future_annotations_or_stub()
                     && checker.settings.target_version < PythonVersion::Py39
-                    && !checker.semantic.future_annotations()
                     && checker.semantic.in_annotation()
                     && typing::is_pep585_generic(value, &checker.semantic)
                 {
@@ -186,10 +184,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                             typing::to_pep585_generic(expr, &checker.semantic)
                         {
                             if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
-                                if !checker.source_type.is_stub()
+                                if !checker.semantic.future_annotations_or_stub()
                                     && checker.settings.target_version < PythonVersion::Py39
                                     && checker.settings.target_version >= PythonVersion::Py37
-                                    && !checker.semantic.future_annotations()
                                     && checker.semantic.in_annotation()
                                     && !checker.settings.pyupgrade.keep_runtime_typing
                                 {
@@ -200,7 +197,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                                 if checker.source_type.is_stub()
                                     || checker.settings.target_version >= PythonVersion::Py39
                                     || (checker.settings.target_version >= PythonVersion::Py37
-                                        && checker.semantic.future_annotations()
+                                        && checker.semantic.future_annotations_or_stub()
                                         && checker.semantic.in_annotation()
                                         && !checker.settings.pyupgrade.keep_runtime_typing)
                                 {
@@ -270,10 +267,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
             ]) {
                 if let Some(replacement) = typing::to_pep585_generic(expr, &checker.semantic) {
                     if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
-                        if !checker.source_type.is_stub()
+                        if !checker.semantic.future_annotations_or_stub()
                             && checker.settings.target_version < PythonVersion::Py39
                             && checker.settings.target_version >= PythonVersion::Py37
-                            && !checker.semantic.future_annotations()
                             && checker.semantic.in_annotation()
                             && !checker.settings.pyupgrade.keep_runtime_typing
                         {
@@ -286,7 +282,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                         if checker.source_type.is_stub()
                             || checker.settings.target_version >= PythonVersion::Py39
                             || (checker.settings.target_version >= PythonVersion::Py37
-                                && checker.semantic.future_annotations()
+                                && checker.semantic.future_annotations_or_stub()
                                 && checker.semantic.in_annotation()
                                 && !checker.settings.pyupgrade.keep_runtime_typing)
                         {
@@ -1176,9 +1172,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
         }) => {
             // Ex) `str | None`
             if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
-                if !checker.source_type.is_stub()
+                if !checker.semantic.future_annotations_or_stub()
                     && checker.settings.target_version < PythonVersion::Py310
-                    && !checker.semantic.future_annotations()
                     && checker.semantic.in_annotation()
                 {
                     flake8_future_annotations::rules::future_required_type_annotation(

crates/ruff_linter/src/checkers/ast/annotation.rs

@@ -56,9 +56,10 @@ impl AnnotationContext {
             _ => {}
         }
 
-        // If `__future__` annotations are enabled, then annotations are never evaluated
+        // If `__future__` annotations are enabled or it's a stub file,
-        // at runtime, so we can treat them as typing-only.
+        // then annotations are never evaluated at runtime,
-        if semantic.future_annotations() {
+        // so we can treat them as typing-only.
+        if semantic.future_annotations_or_stub() {
             return Self::TypingOnly;
         }
 
@@ -87,7 +88,7 @@ impl AnnotationContext {
             semantic,
         ) {
             Self::RuntimeRequired
-        } else if semantic.future_annotations() {
+        } else if semantic.future_annotations_or_stub() {
             Self::TypingOnly
         } else {
             Self::RuntimeEvaluated

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -935,9 +935,12 @@ impl<'a> Visitor<'a> for Checker<'a> {
     fn visit_expr(&mut self, expr: &'a Expr) {
         // Step 0: Pre-processing
         if !self.semantic.in_typing_literal()
+            // `in_deferred_type_definition()` will only be `true` if we're now visiting the deferred nodes
+            // after having already traversed the source tree once. If we're now visiting the deferred nodes,
+            // we can't defer again, or we'll infinitely recurse!
             && !self.semantic.in_deferred_type_definition()
             && self.semantic.in_type_definition()
-            && self.semantic.future_annotations()
+            && self.semantic.future_annotations_or_stub()
             && (self.semantic.in_annotation() || self.source_type.is_stub())
         {
             if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = expr {
@@ -1964,6 +1967,25 @@ impl<'a> Checker<'a> {
         scope.add(id, binding_id);
     }
 
+    /// After initial traversal of the AST, visit all "future type definitions".
+    ///
+    /// A "future type definition" is a type definition where [PEP 563] semantics
+    /// apply (i.e., an annotation in a module that has `from __future__ import annotations`
+    /// at the top of the file, or an annotation in a stub file). These type definitions
+    /// support forward references, so they are deferred on initial traversal
+    /// of the source tree.
+    ///
+    /// For example:
+    /// ```python
+    /// from __future__ import annotations
+    ///
+    /// def foo() -> Bar:  # <-- return annotation is a "future type definition"
+    ///     return Bar()
+    ///
+    /// class Bar: pass
+    /// ```
+    ///
+    /// [PEP 563]: https://peps.python.org/pep-0563/
     fn visit_deferred_future_type_definitions(&mut self) {
         let snapshot = self.semantic.snapshot();
         while !self.visit.future_type_definitions.is_empty() {
@@ -1971,6 +1993,14 @@ impl<'a> Checker<'a> {
             for (expr, snapshot) in type_definitions {
                 self.semantic.restore(snapshot);
 
+                // Type definitions should only be considered "`__future__` type definitions"
+                // if they are annotations in a module where `from __future__ import
+                // annotations` is active, or they are type definitions in a stub file.
+                debug_assert!(
+                    self.semantic.future_annotations_or_stub()
+                        && (self.source_type.is_stub() || self.semantic.in_annotation())
+                );
+
                 self.semantic.flags |= SemanticModelFlags::TYPE_DEFINITION
                     | SemanticModelFlags::FUTURE_TYPE_DEFINITION;
                 self.visit_expr(expr);
@@ -1979,6 +2009,19 @@ impl<'a> Checker<'a> {
         self.semantic.restore(snapshot);
     }
 
+    /// After initial traversal of the AST, visit all [type parameter definitions].
+    ///
+    /// Type parameters natively support forward references,
+    /// so are always deferred during initial traversal of the source tree.
+    ///
+    /// For example:
+    /// ```python
+    /// class Foo[T: Bar]: pass  # <-- Forward reference used in definition of type parameter `T`
+    /// type X[T: Bar] = Foo[T]  # <-- Ditto
+    /// class Bar: pass
+    /// ```
+    ///
+    /// [type parameter definitions]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes
     fn visit_deferred_type_param_definitions(&mut self) {
         let snapshot = self.semantic.snapshot();
         while !self.visit.type_param_definitions.is_empty() {
@@ -1994,6 +2037,17 @@ impl<'a> Checker<'a> {
         self.semantic.restore(snapshot);
     }
 
+    /// After initial traversal of the AST, visit all "string type definitions",
+    /// i.e., type definitions that are enclosed within quotes so as to allow
+    /// the type definition to use forward references.
+    ///
+    /// For example:
+    /// ```python
+    /// def foo() -> "Bar":  # <-- return annotation is a "string type definition"
+    ///     return Bar()
+    ///
+    /// class Bar: pass
+    /// ```
     fn visit_deferred_string_type_definitions(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
         let snapshot = self.semantic.snapshot();
         while !self.visit.string_type_definitions.is_empty() {
@@ -2006,7 +2060,7 @@ impl<'a> Checker<'a> {
 
                     self.semantic.restore(snapshot);
 
-                    if self.semantic.in_annotation() && self.semantic.future_annotations() {
+                    if self.semantic.in_annotation() && self.semantic.future_annotations_or_stub() {
                         if self.enabled(Rule::QuotedAnnotation) {
                             pyupgrade::rules::quoted_annotation(self, value, range);
                         }
@@ -2042,6 +2096,11 @@ impl<'a> Checker<'a> {
         self.semantic.restore(snapshot);
     }
 
+    /// After initial traversal of the AST, visit all function bodies.
+    ///
+    /// Function bodies are always deferred on initial traversal of the source tree,
+    /// as the body of a function may validly contain references to global-scope symbols
+    /// that were not yet defined at the point when the function was defined.
     fn visit_deferred_functions(&mut self) {
         let snapshot = self.semantic.snapshot();
         while !self.visit.functions.is_empty() {
@@ -2065,8 +2124,9 @@ impl<'a> Checker<'a> {
         self.semantic.restore(snapshot);
     }
 
-    /// Visit all deferred lambdas. Returns a list of snapshots, such that the caller can restore
+    /// After initial traversal of the source tree has been completed,
-    /// the semantic model to the state it was in before visiting the deferred lambdas.
+    /// visit all lambdas. Lambdas are deferred during the initial traversal
+    /// for the same reason as function bodies.
     fn visit_deferred_lambdas(&mut self) {
         let snapshot = self.semantic.snapshot();
         while !self.visit.lambdas.is_empty() {
@@ -2092,8 +2152,9 @@ impl<'a> Checker<'a> {
         self.semantic.restore(snapshot);
     }
 
-    /// Recursively visit all deferred AST nodes, including lambdas, functions, and type
+    /// After initial traversal of the source tree has been completed,
-    /// annotations.
+    /// recursively visit all AST nodes that were deferred on the first pass.
+    /// This includes lambdas, functions, type parameters, and type annotations.
     fn visit_deferred(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
         while !self.visit.is_empty() {
             self.visit_deferred_functions();

crates/ruff_python_semantic/src/model.rs

@@ -1465,31 +1465,36 @@ impl<'a> SemanticModel<'a> {
         self.flags.intersects(SemanticModelFlags::TYPE_DEFINITION)
     }
 
-    /// Return `true` if the model is in a string type definition.
+    /// Return `true` if the model is visiting a "string type definition"
+    /// that was previously deferred when initially traversing the AST
     pub const fn in_string_type_definition(&self) -> bool {
         self.flags
             .intersects(SemanticModelFlags::STRING_TYPE_DEFINITION)
     }
 
-    /// Return `true` if the model is in a "simple" string type definition.
+    /// Return `true` if the model is visiting a "simple string type definition"
+    /// that was previously deferred when initially traversing the AST
     pub const fn in_simple_string_type_definition(&self) -> bool {
         self.flags
             .intersects(SemanticModelFlags::SIMPLE_STRING_TYPE_DEFINITION)
     }
 
-    /// Return `true` if the model is in a "complex" string type definition.
+    /// Return `true` if the model is visiting a "complex string type definition"
+    /// that was previously deferred when initially traversing the AST
     pub const fn in_complex_string_type_definition(&self) -> bool {
         self.flags
             .intersects(SemanticModelFlags::COMPLEX_STRING_TYPE_DEFINITION)
     }
 
-    /// Return `true` if the model is in a `__future__` type definition.
+    /// Return `true` if the model is visiting a "`__future__` type definition"
+    /// that was previously deferred when initially traversing the AST
     pub const fn in_future_type_definition(&self) -> bool {
         self.flags
             .intersects(SemanticModelFlags::FUTURE_TYPE_DEFINITION)
     }
 
-    /// Return `true` if the model is in any kind of deferred type definition.
+    /// Return `true` if the model is visiting any kind of type definition
+    /// that was previously deferred when initially traversing the AST
     pub const fn in_deferred_type_definition(&self) -> bool {
         self.flags
             .intersects(SemanticModelFlags::DEFERRED_TYPE_DEFINITION)
@@ -1574,9 +1579,9 @@ impl<'a> SemanticModel<'a> {
     }
 
     /// Return `true` if `__future__`-style type annotations are enabled.
-    pub const fn future_annotations(&self) -> bool {
+    pub const fn future_annotations_or_stub(&self) -> bool {
         self.flags
-            .intersects(SemanticModelFlags::FUTURE_ANNOTATIONS)
+            .intersects(SemanticModelFlags::FUTURE_ANNOTATIONS_OR_STUB)
     }
 
     /// Return `true` if the model is in a stub file (i.e., a file with a `.pyi` extension).
@@ -1770,6 +1775,9 @@ bitflags! {
         ///
         /// "Simple" string type definitions are those that consist of a single string literal,
         /// as opposed to an implicitly concatenated string literal.
+        ///
+        /// Note that this flag is only set when we are actually *visiting* the deferred definition,
+        /// not when we "pass by" it when initially traversing the source tree.
         const SIMPLE_STRING_TYPE_DEFINITION =  1 << 4;
 
         /// The model is in a (deferred) "complex" string type definition.
@@ -1781,6 +1789,9 @@ bitflags! {
         ///
         /// "Complex" string type definitions are those that consist of a implicitly concatenated
         /// string literals. These are uncommon but valid.
+        ///
+        /// Note that this flag is only set when we are actually *visiting* the deferred definition,
+        /// not when we "pass by" it when initially traversing the source tree.
         const COMPLEX_STRING_TYPE_DEFINITION = 1 << 5;
 
         /// The model is in a (deferred) `__future__` type definition.
@@ -1794,6 +1805,20 @@ bitflags! {
         ///
         /// `__future__`-style type annotations are only enabled if the `annotations` feature
         /// is enabled via `from __future__ import annotations`.
+        ///
+        /// This flag should only be set in contexts where PEP-563 semantics are relevant to
+        /// resolution of the type definition. For example, the flag should not be set
+        /// in the following context, because the type definition is not inside a type annotation,
+        /// so whether or not `from __future__ import annotations` is active has no relevance:
+        /// ```python
+        /// from __future__ import annotations
+        /// from typing import TypeAlias
+        ///
+        /// X: TypeAlias = list[int]
+        /// ```
+        ///
+        /// Note also that this flag is only set when we are actually *visiting* the deferred definition,
+        /// not when we "pass by" it when initially traversing the source tree.
         const FUTURE_TYPE_DEFINITION = 1 << 6;
 
         /// The model is in an exception handler.
@@ -1884,7 +1909,8 @@ bitflags! {
         /// any other non-`__future__`-importing statements.
         const FUTURES_BOUNDARY = 1 << 14;
 
-        /// `__future__`-style type annotations are enabled in this model.
+        /// The model is in a file that has `from __future__ import annotations`
+        /// at the top of the module.
         ///
         /// For example, the model could be visiting `x` in:
         /// ```python
@@ -1899,6 +1925,12 @@ bitflags! {
         /// The model is in a Python stub file (i.e., a `.pyi` file).
         const STUB_FILE = 1 << 16;
 
+        /// `__future__`-style type annotations are enabled in this model.
+        /// That could be because it's a stub file,
+        /// or it could be because it's a non-stub file that has `from __future__ import annotations`
+        /// a the top of the module.
+        const FUTURE_ANNOTATIONS_OR_STUB = Self::FUTURE_ANNOTATIONS.bits() | Self::STUB_FILE.bits();
+
         /// The model has traversed past the module docstring.
         ///
         /// For example, the model could be visiting `x` in:
@@ -1909,14 +1941,28 @@ bitflags! {
         /// ```
         const MODULE_DOCSTRING_BOUNDARY = 1 << 17;
 
-        /// The model is in a type parameter definition.
+        /// The model is in a (deferred) [type parameter definition].
         ///
-        /// For example, the model could be visiting `Record` in:
+        /// For example, the model could be visiting `T`, `P` or `Ts` in:
         /// ```python
-        /// from typing import TypeVar
+        /// class Foo[T, *Ts, **P]: pass
+        /// ```
         ///
-        /// Record = TypeVar("Record")
+        /// Note that this flag is *not* set for "pre-PEP-695" TypeVars, ParamSpecs or TypeVarTuples.
+        /// None of the following would lead to the flag being set:
         ///
+        /// ```python
+        /// from typing import TypeVar, ParamSpec, TypeVarTuple
+        ///
+        /// T = TypeVar("T")
+        /// P = ParamSpec("P")
+        /// Ts = TypeVarTuple("Ts")
+        /// ```
+        ///
+        /// Note also that this flag is only set when we are actually *visiting* the deferred definition,
+        /// not when we "pass by" it when initially traversing the source tree.
+        ///
+        /// [type parameter definition]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes
         const TYPE_PARAM_DEFINITION = 1 << 18;
 
         /// The model is in a named expression assignment.
@@ -1996,12 +2042,11 @@ bitflags! {
 
 impl SemanticModelFlags {
     pub fn new(path: &Path) -> Self {
-        let mut flags = Self::default();
         if is_python_stub_file(path) {
-            flags |= Self::STUB_FILE;
+            Self::STUB_FILE
-            flags |= Self::FUTURE_ANNOTATIONS;
+        } else {
+            Self::default()
         }
-        flags
     }
 }
 

crates/ruff_python_semantic/src/scope.rs

@@ -185,6 +185,7 @@ pub enum ScopeKind<'a> {
     Function(&'a ast::StmtFunctionDef),
     Generator,
     Module,
+    /// A Python 3.12+ ["annotation scope"](https://docs.python.org/3/reference/executionmodel.html#annotation-scopes)
     Type,
     Lambda(&'a ast::ExprLambda),
 }