diff --git a/Cargo.lock b/Cargo.lock index e6777f13ec..4911afc3d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2851,7 +2851,6 @@ dependencies = [ "anstyle", "arc-swap", "camino", - "countme", "dashmap", "dunce", "etcetera", diff --git a/crates/ruff_db/Cargo.toml b/crates/ruff_db/Cargo.toml index 9a446a1b19..ccc9edf480 100644 --- a/crates/ruff_db/Cargo.toml +++ b/crates/ruff_db/Cargo.toml @@ -25,7 +25,6 @@ ty_static = { workspace = true } anstyle = { workspace = true } arc-swap = { workspace = true } camino = { workspace = true } -countme = { workspace = true } dashmap = { workspace = true } dunce = { workspace = true } filetime = { workspace = true } @@ -59,6 +58,11 @@ tempfile = { workspace = true } cache = ["ruff_cache"] junit = ["dep:quick-junit"] os = ["ignore", "dep:etcetera"] -serde = ["camino/serde1", "dep:serde", "dep:serde_json", "ruff_diagnostics/serde"] +serde = [ + "camino/serde1", + "dep:serde", + "dep:serde_json", + "ruff_diagnostics/serde", +] # Exposes testing utilities. testing = ["tracing-subscriber"] diff --git a/crates/ruff_db/src/files.rs b/crates/ruff_db/src/files.rs index 37400e2c2a..9393ee4b7c 100644 --- a/crates/ruff_db/src/files.rs +++ b/crates/ruff_db/src/files.rs @@ -1,7 +1,6 @@ use std::fmt; use std::sync::Arc; -use countme::Count; use dashmap::mapref::entry::Entry; pub use file_root::{FileRoot, FileRootKind}; pub use path::FilePath; @@ -312,11 +311,6 @@ pub struct File { /// the file has been deleted is to change the status to `Deleted`. #[default] status: FileStatus, - - /// Counter that counts the number of created file instances and active file instances. - /// Only enabled in debug builds. - #[default] - count: Count, } // The Salsa heap is tracked separately. diff --git a/crates/ruff_db/src/source.rs b/crates/ruff_db/src/source.rs index 824c17cae4..ca303e6bbd 100644 --- a/crates/ruff_db/src/source.rs +++ b/crates/ruff_db/src/source.rs @@ -1,8 +1,6 @@ use std::ops::Deref; use std::sync::Arc; -use countme::Count; - use ruff_notebook::Notebook; use ruff_python_ast::PySourceType; use ruff_source_file::LineIndex; @@ -38,11 +36,7 @@ pub fn source_text(db: &dyn Db, file: File) -> SourceText { }; SourceText { - inner: Arc::new(SourceTextInner { - kind, - read_error, - count: Count::new(), - }), + inner: Arc::new(SourceTextInner { kind, read_error }), } } @@ -125,8 +119,6 @@ impl std::fmt::Debug for SourceText { #[derive(Eq, PartialEq, get_size2::GetSize)] struct SourceTextInner { - #[get_size(ignore)] - count: Count, kind: SourceTextKind, read_error: Option, } diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 76c33a47ea..b5b7933a77 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -104,7 +104,10 @@ pub fn check_types(db: &dyn Db, file: File) -> Vec { for scope_id in index.scope_ids() { let result = infer_scope_types(db, scope_id); - diagnostics.extend(result.diagnostics()); + + if let Some(scope_diagnostics) = result.diagnostics() { + diagnostics.extend(scope_diagnostics); + } } diagnostics.extend_diagnostics( @@ -116,7 +119,7 @@ pub fn check_types(db: &dyn Db, file: File) -> Vec { check_suppressions(db, file, &mut diagnostics); - diagnostics.into_vec() + diagnostics.into_diagnostics() } /// Infer the type of a binding. diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index 34d072638a..b7205b69d7 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -32,7 +32,7 @@ use crate::{ /// ## Consuming /// It's important that the context is explicitly consumed before dropping by calling /// [`InferContext::finish`] and the returned diagnostics must be stored -/// on the current [`TypeInference`](super::infer::TypeInference) result. +/// on the current [`TypeInferenceBuilder`](super::infer::TypeInferenceBuilder) result. pub(crate) struct InferContext<'db, 'ast> { db: &'db dyn Db, scope: ScopeId<'db>, diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 5d4c89ee11..d1d499ca8f 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -1671,18 +1671,26 @@ impl TypeCheckDiagnostics { self.diagnostics.shrink_to_fit(); } - pub(crate) fn into_vec(self) -> Vec { + pub(crate) fn into_diagnostics(self) -> Vec { self.diagnostics } + pub(crate) fn is_empty(&self) -> bool { + self.diagnostics.is_empty() && self.used_suppressions.is_empty() + } + pub fn iter(&self) -> std::slice::Iter<'_, Diagnostic> { - self.diagnostics.iter() + self.diagnostics().iter() + } + + fn diagnostics(&self) -> &[Diagnostic] { + self.diagnostics.as_slice() } } impl std::fmt::Debug for TypeCheckDiagnostics { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - self.diagnostics.fmt(f) + self.diagnostics().fmt(f) } } @@ -1691,7 +1699,7 @@ impl IntoIterator for TypeCheckDiagnostics { type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.diagnostics.into_iter() + self.into_diagnostics().into_iter() } } @@ -1699,8 +1707,9 @@ impl<'a> IntoIterator for &'a TypeCheckDiagnostics { type Item = &'a Diagnostic; type IntoIter = std::slice::Iter<'a, Diagnostic>; + #[inline] fn into_iter(self) -> Self::IntoIter { - self.diagnostics.iter() + self.iter() } } diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index 6bbfe74f80..b0ab2b8492 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -18,8 +18,10 @@ //! cached by Salsa. We also need the expression-level query for inferring types in type guard //! expressions (e.g. the test clause of an `if` statement.) //! -//! Inferring types at any of the three region granularities returns a [`TypeInference`], which -//! holds types for every [`Definition`] and expression within the inferred region. +//! Inferring types at any of the three region granularities returns a [`ExpressionInference`], +//! [`DefinitionInference`], or [`ScopeInference`], which hold the types for every expression +//! within the inferred region. Some inference types also expose the type of every definition +//! within the inferred region. //! //! Some type expressions can require deferred evaluation. This includes all type expressions in //! stub files, or annotation expressions in modules with `from __future__ import annotations`, or @@ -28,7 +30,7 @@ //! definitions once the rest of the types in the scope have been inferred. //! //! Many of our type inference Salsa queries implement cycle recovery via fixed-point iteration. In -//! general, they initiate fixed-point iteration by returning a `TypeInference` that returns +//! general, they initiate fixed-point iteration by returning an `Inference` type that returns //! `Type::Never` for all expressions, bindings, and declarations, and then they continue iterating //! the query cycle until a fixed-point is reached. Salsa has a built-in fixed limit on the number //! of iterations, so if we fail to converge, Salsa will eventually panic. (This should of course @@ -127,7 +129,7 @@ use crate::{Db, FxOrderSet, Program}; /// Use when checking a scope, or needing to provide a type for an arbitrary expression in the /// scope. #[salsa::tracked(returns(ref), cycle_fn=scope_cycle_recover, cycle_initial=scope_cycle_initial, heap_size=get_size2::GetSize::get_heap_size)] -pub(crate) fn infer_scope_types<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> TypeInference<'db> { +pub(crate) fn infer_scope_types<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> ScopeInference<'db> { let file = scope.file(db); let _span = tracing::trace_span!("infer_scope_types", scope=?scope.as_id(), ?file).entered(); @@ -137,20 +139,20 @@ pub(crate) fn infer_scope_types<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Ty // The isolation of the query is by the return inferred types. let index = semantic_index(db, file); - TypeInferenceBuilder::new(db, InferenceRegion::Scope(scope), index, &module).finish() + TypeInferenceBuilder::new(db, InferenceRegion::Scope(scope), index, &module).finish_scope() } fn scope_cycle_recover<'db>( _db: &'db dyn Db, - _value: &TypeInference<'db>, + _value: &ScopeInference<'db>, _count: u32, _scope: ScopeId<'db>, -) -> salsa::CycleRecoveryAction> { +) -> salsa::CycleRecoveryAction> { salsa::CycleRecoveryAction::Iterate } -fn scope_cycle_initial<'db>(_db: &'db dyn Db, scope: ScopeId<'db>) -> TypeInference<'db> { - TypeInference::cycle_fallback(scope, Type::Never) +fn scope_cycle_initial<'db>(_db: &'db dyn Db, scope: ScopeId<'db>) -> ScopeInference<'db> { + ScopeInference::cycle_fallback(scope) } /// Infer all types for a [`Definition`] (including sub-expressions). @@ -159,7 +161,7 @@ fn scope_cycle_initial<'db>(_db: &'db dyn Db, scope: ScopeId<'db>) -> TypeInfere pub(crate) fn infer_definition_types<'db>( db: &'db dyn Db, definition: Definition<'db>, -) -> TypeInference<'db> { +) -> DefinitionInference<'db> { let file = definition.file(db); let module = parsed_module(db, file).load(db); let _span = tracing::trace_span!( @@ -171,23 +173,24 @@ pub(crate) fn infer_definition_types<'db>( let index = semantic_index(db, file); - TypeInferenceBuilder::new(db, InferenceRegion::Definition(definition), index, &module).finish() + TypeInferenceBuilder::new(db, InferenceRegion::Definition(definition), index, &module) + .finish_definition() } fn definition_cycle_recover<'db>( _db: &'db dyn Db, - _value: &TypeInference<'db>, + _value: &DefinitionInference<'db>, _count: u32, _definition: Definition<'db>, -) -> salsa::CycleRecoveryAction> { +) -> salsa::CycleRecoveryAction> { salsa::CycleRecoveryAction::Iterate } fn definition_cycle_initial<'db>( db: &'db dyn Db, definition: Definition<'db>, -) -> TypeInference<'db> { - TypeInference::cycle_fallback(definition.scope(db), Type::Never) +) -> DefinitionInference<'db> { + DefinitionInference::cycle_fallback(definition.scope(db)) } /// Infer types for all deferred type expressions in a [`Definition`]. @@ -198,7 +201,7 @@ fn definition_cycle_initial<'db>( pub(crate) fn infer_deferred_types<'db>( db: &'db dyn Db, definition: Definition<'db>, -) -> TypeInference<'db> { +) -> DefinitionInference<'db> { let file = definition.file(db); let module = parsed_module(db, file).load(db); let _span = tracing::trace_span!( @@ -211,20 +214,24 @@ pub(crate) fn infer_deferred_types<'db>( let index = semantic_index(db, file); - TypeInferenceBuilder::new(db, InferenceRegion::Deferred(definition), index, &module).finish() + TypeInferenceBuilder::new(db, InferenceRegion::Deferred(definition), index, &module) + .finish_definition() } fn deferred_cycle_recover<'db>( _db: &'db dyn Db, - _value: &TypeInference<'db>, + _value: &DefinitionInference<'db>, _count: u32, _definition: Definition<'db>, -) -> salsa::CycleRecoveryAction> { +) -> salsa::CycleRecoveryAction> { salsa::CycleRecoveryAction::Iterate } -fn deferred_cycle_initial<'db>(db: &'db dyn Db, definition: Definition<'db>) -> TypeInference<'db> { - TypeInference::cycle_fallback(definition.scope(db), Type::Never) +fn deferred_cycle_initial<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> DefinitionInference<'db> { + DefinitionInference::cycle_fallback(definition.scope(db)) } /// Infer all types for an [`Expression`] (including sub-expressions). @@ -235,7 +242,7 @@ fn deferred_cycle_initial<'db>(db: &'db dyn Db, definition: Definition<'db>) -> pub(crate) fn infer_expression_types<'db>( db: &'db dyn Db, expression: Expression<'db>, -) -> TypeInference<'db> { +) -> ExpressionInference<'db> { let file = expression.file(db); let module = parsed_module(db, file).load(db); let _span = tracing::trace_span!( @@ -248,23 +255,24 @@ pub(crate) fn infer_expression_types<'db>( let index = semantic_index(db, file); - TypeInferenceBuilder::new(db, InferenceRegion::Expression(expression), index, &module).finish() + TypeInferenceBuilder::new(db, InferenceRegion::Expression(expression), index, &module) + .finish_expression() } fn expression_cycle_recover<'db>( _db: &'db dyn Db, - _value: &TypeInference<'db>, + _value: &ExpressionInference<'db>, _count: u32, _expression: Expression<'db>, -) -> salsa::CycleRecoveryAction> { +) -> salsa::CycleRecoveryAction> { salsa::CycleRecoveryAction::Iterate } fn expression_cycle_initial<'db>( db: &'db dyn Db, expression: Expression<'db>, -) -> TypeInference<'db> { - TypeInference::cycle_fallback(expression.scope(db), Type::Never) +) -> ExpressionInference<'db> { + ExpressionInference::cycle_fallback(expression.scope(db)) } /// Infers the type of an `expression` that is guaranteed to be in the same file as the calling query. @@ -405,55 +413,130 @@ struct TypeAndRange<'db> { range: TextRange, } -/// The inferred types for a single region. +/// The inferred types for a scope region. #[derive(Debug, Eq, PartialEq, salsa::Update, get_size2::GetSize)] -pub(crate) struct TypeInference<'db> { +pub(crate) struct ScopeInference<'db> { /// The types of every expression in this region. expressions: FxHashMap>, - /// The types of every binding in this region. - bindings: FxHashMap, Type<'db>>, - - /// The types and type qualifiers of every declaration in this region. - declarations: FxHashMap, TypeAndQualifiers<'db>>, - - /// The definitions that are deferred. - deferred: FxHashSet>, - - /// The diagnostics for this region. - diagnostics: TypeCheckDiagnostics, - - /// The scope this region is part of. - scope: ScopeId<'db>, + /// The extra data that is only present for few inference regions. + extra: Option>, +} +#[derive(Debug, Eq, PartialEq, get_size2::GetSize, salsa::Update, Default)] +struct ScopeInferenceExtra { /// The fallback type for missing expressions/bindings/declarations. /// /// This is used only when constructing a cycle-recovery `TypeInference`. - cycle_fallback_type: Option>, + cycle_fallback: bool, + + /// The diagnostics for this region. + diagnostics: TypeCheckDiagnostics, } -impl<'db> TypeInference<'db> { - pub(crate) fn empty(scope: ScopeId<'db>) -> Self { +impl<'db> ScopeInference<'db> { + fn cycle_fallback(scope: ScopeId<'db>) -> Self { + let _ = scope; + Self { + extra: Some(Box::new(ScopeInferenceExtra { + cycle_fallback: true, + ..ScopeInferenceExtra::default() + })), expressions: FxHashMap::default(), - bindings: FxHashMap::default(), - declarations: FxHashMap::default(), - deferred: FxHashSet::default(), - diagnostics: TypeCheckDiagnostics::default(), - scope, - cycle_fallback_type: None, } } - fn cycle_fallback(scope: ScopeId<'db>, cycle_fallback_type: Type<'db>) -> Self { + pub(crate) fn diagnostics(&self) -> Option<&TypeCheckDiagnostics> { + self.extra.as_deref().map(|extra| &extra.diagnostics) + } + + #[track_caller] + pub(crate) fn expression_type(&self, expression: impl Into) -> Type<'db> { + self.try_expression_type(expression).expect( + "Failed to retrieve the inferred type for an `ast::Expr` node \ + passed to `TypeInference::expression_type()`. The `TypeInferenceBuilder` \ + should infer and store types for all `ast::Expr` nodes in any `TypeInference` \ + region it analyzes.", + ) + } + + pub(crate) fn try_expression_type( + &self, + expression: impl Into, + ) -> Option> { + self.expressions + .get(&expression.into()) + .copied() + .or_else(|| self.fallback_type()) + } + + fn is_cycle_callback(&self) -> bool { + self.extra + .as_ref() + .is_some_and(|extra| extra.cycle_fallback) + } + + fn fallback_type(&self) -> Option> { + self.is_cycle_callback().then_some(Type::Never) + } +} + +/// The inferred types for a definition region. +#[derive(Debug, Eq, PartialEq, salsa::Update, get_size2::GetSize)] +pub(crate) struct DefinitionInference<'db> { + /// The types of every expression in this region. + expressions: FxHashMap>, + + /// The scope this region is part of. + #[cfg(debug_assertions)] + scope: ScopeId<'db>, + + /// The types of every binding in this region. + /// + /// Almost all definition regions have less than 10 bindings. There are very few with more than 10 (but still less than 20). + /// Because of that, use a slice with linear search over a hash map. + bindings: Box<[(Definition<'db>, Type<'db>)]>, + + /// The types and type qualifiers of every declaration in this region. + /// + /// About 50% of the definition inference regions have no declarations. + /// The other 50% have less than 10 declarations. Because of that, use a + /// slice with linear search over a hash map. + declarations: Box<[(Definition<'db>, TypeAndQualifiers<'db>)]>, + + /// The extra data that is only present for few inference regions. + extra: Option>>, +} + +#[derive(Debug, Eq, PartialEq, get_size2::GetSize, salsa::Update, Default)] +struct DefinitionInferenceExtra<'db> { + /// The fallback type for missing expressions/bindings/declarations. + /// + /// This is used only when constructing a cycle-recovery `TypeInference`. + cycle_fallback: bool, + + /// The definitions that are deferred. + deferred: Box<[Definition<'db>]>, + + /// The diagnostics for this region. + diagnostics: TypeCheckDiagnostics, +} + +impl<'db> DefinitionInference<'db> { + fn cycle_fallback(scope: ScopeId<'db>) -> Self { + let _ = scope; + Self { expressions: FxHashMap::default(), - bindings: FxHashMap::default(), - declarations: FxHashMap::default(), - deferred: FxHashSet::default(), - diagnostics: TypeCheckDiagnostics::default(), + bindings: Box::default(), + declarations: Box::default(), + #[cfg(debug_assertions)] scope, - cycle_fallback_type: Some(cycle_fallback_type), + extra: Some(Box::new(DefinitionInferenceExtra { + cycle_fallback: true, + ..DefinitionInferenceExtra::default() + })), } } @@ -474,43 +557,141 @@ impl<'db> TypeInference<'db> { self.expressions .get(&expression.into()) .copied() - .or(self.cycle_fallback_type) + .or_else(|| self.fallback_type()) } #[track_caller] pub(crate) fn binding_type(&self, definition: Definition<'db>) -> Type<'db> { self.bindings - .get(&definition) - .copied() - .or(self.cycle_fallback_type) + .iter() + .find_map( + |(def, ty)| { + if def == &definition { Some(*ty) } else { None } + }, + ) + .or_else(|| self.fallback_type()) .expect( "definition should belong to this TypeInference region and \ TypeInferenceBuilder should have inferred a type for it", ) } + fn bindings(&self) -> impl ExactSizeIterator, Type<'db>)> { + self.bindings.iter().copied() + } + #[track_caller] pub(crate) fn declaration_type(&self, definition: Definition<'db>) -> TypeAndQualifiers<'db> { self.declarations - .get(&definition) - .copied() - .or(self.cycle_fallback_type.map(Into::into)) + .iter() + .find_map(|(def, qualifiers)| { + if def == &definition { + Some(*qualifiers) + } else { + None + } + }) + .or_else(|| self.fallback_type().map(Into::into)) .expect( "definition should belong to this TypeInference region and \ TypeInferenceBuilder should have inferred a type for it", ) } - pub(crate) fn diagnostics(&self) -> &TypeCheckDiagnostics { - &self.diagnostics + fn declarations( + &self, + ) -> impl ExactSizeIterator, TypeAndQualifiers<'db>)> { + self.declarations.iter().copied() } - fn shrink_to_fit(&mut self) { - self.expressions.shrink_to_fit(); - self.bindings.shrink_to_fit(); - self.declarations.shrink_to_fit(); - self.diagnostics.shrink_to_fit(); - self.deferred.shrink_to_fit(); + fn declaration_types(&self) -> impl ExactSizeIterator> { + self.declarations.iter().map(|(_, qualifiers)| *qualifiers) + } + + fn is_cycle_callback(&self) -> bool { + self.extra + .as_ref() + .is_some_and(|extra| extra.cycle_fallback) + } + + fn fallback_type(&self) -> Option> { + self.is_cycle_callback().then_some(Type::Never) + } +} + +/// The inferred types for an expression region. +#[derive(Debug, Eq, PartialEq, salsa::Update, get_size2::GetSize)] +pub(crate) struct ExpressionInference<'db> { + /// The types of every expression in this region. + expressions: FxHashMap>, + + extra: Option>>, + + /// The scope this region is part of. + #[cfg(debug_assertions)] + scope: ScopeId<'db>, +} + +/// Extra data that only exists for few inferred expression regions. +#[derive(Debug, Eq, PartialEq, salsa::Update, get_size2::GetSize, Default)] +struct ExpressionInferenceExtra<'db> { + /// The types of every binding in this expression region. + /// + /// Only very few expression regions have bindings (around 0.1%). + bindings: Box<[(Definition<'db>, Type<'db>)]>, + + /// The diagnostics for this region. + diagnostics: TypeCheckDiagnostics, + + /// `true` if this region is part of a cycle-recovery `TypeInference`. + /// + /// Falls back to `Type::Never` if an expression is missing. + cycle_fallback: bool, +} + +impl<'db> ExpressionInference<'db> { + fn cycle_fallback(scope: ScopeId<'db>) -> Self { + let _ = scope; + Self { + extra: Some(Box::new(ExpressionInferenceExtra { + cycle_fallback: true, + ..ExpressionInferenceExtra::default() + })), + expressions: FxHashMap::default(), + + #[cfg(debug_assertions)] + scope, + } + } + + pub(crate) fn try_expression_type( + &self, + expression: impl Into, + ) -> Option> { + self.expressions + .get(&expression.into()) + .copied() + .or_else(|| self.fallback_type()) + } + + #[track_caller] + pub(crate) fn expression_type(&self, expression: impl Into) -> Type<'db> { + self.try_expression_type(expression).expect( + "Failed to retrieve the inferred type for an `ast::Expr` node \ + passed to `TypeInference::expression_type()`. The `TypeInferenceBuilder` \ + should infer and store types for all `ast::Expr` nodes in any `TypeInference` \ + region it analyzes.", + ) + } + + fn is_cycle_callback(&self) -> bool { + self.extra + .as_ref() + .is_some_and(|extra| extra.cycle_fallback) + } + + fn fallback_type(&self) -> Option> { + self.is_cycle_callback().then_some(Type::Never) } } @@ -536,26 +717,26 @@ enum DeclaredAndInferredType<'db> { /// Builder to infer all types in a region. /// /// A builder is used by creating it with [`new()`](TypeInferenceBuilder::new), and then calling -/// [`finish()`](TypeInferenceBuilder::finish) on it, which returns the resulting -/// [`TypeInference`]. +/// [`finish_expression()`](TypeInferenceBuilder::finish_expression), [`finish_definition()`](TypeInferenceBuilder::finish_definition), or [`finish_scope()`](TypeInferenceBuilder::finish_scope) on it, which returns +/// type inference result.. /// /// There are a few different kinds of methods in the type inference builder, and the naming /// distinctions are a bit subtle. /// -/// The `finish` method calls [`infer_region`](TypeInferenceBuilder::infer_region), which delegates +/// The `finish` methods call [`infer_region`](TypeInferenceBuilder::infer_region), which delegates /// to one of [`infer_region_scope`](TypeInferenceBuilder::infer_region_scope), /// [`infer_region_definition`](TypeInferenceBuilder::infer_region_definition), or /// [`infer_region_expression`](TypeInferenceBuilder::infer_region_expression), depending which /// kind of [`InferenceRegion`] we are inferring types for. /// /// Scope inference starts with the scope body, walking all statements and expressions and -/// recording the types of each expression in the [`TypeInference`] result. Most of the methods +/// recording the types of each expression in the inference result. Most of the methods /// here (with names like `infer_*_statement` or `infer_*_expression` or some other node kind) take /// a single AST node and are called as part of this AST visit. /// /// When the visit encounters a node which creates a [`Definition`], we look up the definition in /// the semantic index and call the [`infer_definition_types()`] query on it, which creates another -/// [`TypeInferenceBuilder`] just for that definition, and we merge the returned [`TypeInference`] +/// [`TypeInferenceBuilder`] just for that definition, and we merge the returned inference result /// into the one we are currently building for the entire scope. Using the query in this way /// ensures that if we first infer types for some scattered definitions in a scope, and later for /// the entire scope, we don't re-infer any types, we reuse the cached inference for those @@ -584,8 +765,27 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { index: &'db SemanticIndex<'db>, region: InferenceRegion<'db>, - /// The type inference results - types: TypeInference<'db>, + /// The types of every expression in this region. + expressions: FxHashMap>, + + /// The scope this region is part of. + scope: ScopeId<'db>, + + // bindings, declarations, and deferred can only exist in definition, or scope contexts. + /// The types of every binding in this region. + /// + /// The list should only contain one entry per binding at most. + bindings: VecMap, Type<'db>>, + + /// The types and type qualifiers of every declaration in this region. + /// + /// The list should only contain one entry per declaration at most. + declarations: VecMap, TypeAndQualifiers<'db>>, + + /// The definitions that are deferred. + /// + /// The list should only contain one entry per deferred. + deferred: VecSet>, /// The returned types and their corresponding ranges of the region, if it is a function body. return_types_and_ranges: Vec>, @@ -625,6 +825,11 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { /// expression could be deferred if the file has `from __future__ import annotations` import or /// is a stub file but we're still in a non-deferred region. deferred_state: DeferredExpressionState, + + /// The fallback type for missing expressions/bindings/declarations. + /// + /// This is used only when constructing a cycle-recovery `TypeInference`. + cycle_fallback: bool, } impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { @@ -650,24 +855,51 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return_types_and_ranges: vec![], called_functions: FxHashSet::default(), deferred_state: DeferredExpressionState::None, - types: TypeInference::empty(scope), + scope, + expressions: FxHashMap::default(), + bindings: VecMap::default(), + declarations: VecMap::default(), + deferred: VecSet::default(), + cycle_fallback: false, } } - fn extend(&mut self, inference: &TypeInference<'db>) { - debug_assert_eq!(self.types.scope, inference.scope); + fn extend_definition(&mut self, inference: &DefinitionInference<'db>) { + #[cfg(debug_assertions)] + assert_eq!(self.scope, inference.scope); - self.types.bindings.extend(inference.bindings.iter()); - self.types - .declarations - .extend(inference.declarations.iter()); - self.types.expressions.extend(inference.expressions.iter()); - self.types.deferred.extend(inference.deferred.iter()); - self.context.extend(inference.diagnostics()); - self.types.cycle_fallback_type = self - .types - .cycle_fallback_type - .or(inference.cycle_fallback_type); + self.expressions.extend(inference.expressions.iter()); + self.declarations.extend(inference.declarations()); + + if !matches!(self.region, InferenceRegion::Scope(..)) { + self.bindings.extend(inference.bindings()); + } + + if let Some(extra) = &inference.extra { + self.cycle_fallback |= extra.cycle_fallback; + self.context.extend(&extra.diagnostics); + self.deferred.extend(extra.deferred.iter().copied()); + } + } + + fn extend_expression(&mut self, inference: &ExpressionInference<'db>) { + #[cfg(debug_assertions)] + assert_eq!(self.scope, inference.scope); + + self.extend_expression_unchecked(inference); + } + + fn extend_expression_unchecked(&mut self, inference: &ExpressionInference<'db>) { + self.expressions.extend(inference.expressions.iter()); + + if let Some(extra) = &inference.extra { + self.context.extend(&extra.diagnostics); + self.cycle_fallback |= extra.cycle_fallback; + + if !matches!(self.region, InferenceRegion::Scope(..)) { + self.bindings.extend(extra.bindings.iter().copied()); + } + } } fn file(&self) -> File { @@ -683,7 +915,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } fn scope(&self) -> ScopeId<'db> { - self.types.scope + self.scope } /// Are we currently inferring types in file with deferred types? @@ -733,11 +965,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// this node. #[track_caller] fn expression_type(&self, expr: &ast::Expr) -> Type<'db> { - self.types.expression_type(expr) + self.try_expression_type(expr).expect( + "Failed to retrieve the inferred type for an `ast::Expr` node \ + passed to `TypeInference::expression_type()`. The `TypeInferenceBuilder` \ + should infer and store types for all `ast::Expr` nodes in any `TypeInference` \ + region it analyzes.", + ) } fn try_expression_type(&self, expr: &ast::Expr) -> Option> { - self.types.try_expression_type(expr) + self.expressions + .get(&expr.into()) + .copied() + .or(self.cycle_fallback.then_some(Type::Never)) } /// Get the type of an expression from any scope in the same file. @@ -812,11 +1052,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // Infer the deferred types for the definitions here to consider the end-of-scope // semantics. - for definition in std::mem::take(&mut self.types.deferred) { - self.extend(infer_deferred_types(self.db(), definition)); + for definition in std::mem::take(&mut self.deferred) { + self.extend_definition(infer_deferred_types(self.db(), definition)); } + assert!( - self.types.deferred.is_empty(), + self.deferred.is_empty(), "Inferring deferred types should not add more deferred definitions" ); @@ -836,20 +1077,16 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// [method resolution order]: https://docs.python.org/3/glossary.html#term-method-resolution-order /// [metaclass]: https://docs.python.org/3/reference/datamodel.html#metaclasses fn check_class_definitions(&mut self) { - let class_definitions = self - .types - .declarations - .iter() - .filter_map(|(definition, ty)| { - // Filter out class literals that result from imports - if let DefinitionKind::Class(class) = definition.kind(self.db()) { - ty.inner_type() - .into_class_literal() - .map(|class_literal| (class_literal, class.node(self.module()))) - } else { - None - } - }); + let class_definitions = self.declarations.iter().filter_map(|(definition, ty)| { + // Filter out class literals that result from imports + if let DefinitionKind::Class(class) = definition.kind(self.db()) { + ty.inner_type() + .into_class_literal() + .map(|class_literal| (class_literal, class.node(self.module()))) + } else { + None + } + }); // Iterate through all class definitions in this scope. for (class, class_node) in class_definitions { @@ -1175,7 +1412,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // because an overloaded function uses the same place for each of the overloads and the // implementation. let overloaded_function_places: FxHashSet<_> = self - .types .declarations .iter() .filter_map(|(definition, ty)| { @@ -1807,7 +2043,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } - self.types.bindings.insert(binding, bound_ty); + self.bindings.insert(binding, bound_ty); } /// Returns `true` if `symbol_id` should be looked up in the global scope, skipping intervening @@ -1862,7 +2098,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } TypeAndQualifiers::unknown() }; - self.types.declarations.insert(declaration, ty); + self.declarations.insert(declaration, ty); } fn add_declaration_with_binding( @@ -1932,8 +2168,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } }; - self.types.declarations.insert(definition, declared_ty); - self.types.bindings.insert(definition, inferred_ty); + self.declarations.insert(definition, declared_ty); + self.bindings.insert(definition, inferred_ty); } fn add_unknown_declaration_with_binding( @@ -2272,7 +2508,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fn infer_definition(&mut self, node: impl Into + std::fmt::Debug + Copy) { let definition = self.index.expect_single_definition(node); let result = infer_definition_types(self.db(), definition); - self.extend(result); + self.extend_definition(result); } fn infer_function_definition_statement(&mut self, function: &ast::StmtFunctionDef) { @@ -2342,7 +2578,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // `infer_function_type_params`, rather than here. if type_params.is_none() { if self.defer_annotations() { - self.types.deferred.insert(definition); + self.deferred.insert(definition); } else { self.infer_optional_annotation_expression( returns.as_deref(), @@ -2709,7 +2945,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // deferring the entire class definition if a string literal occurs anywhere in the // base class list. if self.in_stub() || class_node.bases().iter().any(contains_string_literal) { - self.types.deferred.insert(definition); + self.deferred.insert(definition); } else { for base in class_node.bases() { self.infer_expression(base); @@ -3990,8 +4226,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // But here we explicitly overwrite the type for the overall `self.attr` node with // the annotated type. We do no use `store_expression_type` here, because it checks // that no type has been stored for the expression before. - self.types - .expressions + self.expressions .insert((&**target).into(), annotated.inner_type()); } } @@ -4421,11 +4656,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let inferred = infer_definition_types(self.db(), *definition); // Check non-star imports for deprecations if definition.kind(self.db()).as_star_import().is_none() { - for ty in inferred.declarations.values() { + for ty in inferred.declaration_types() { self.check_deprecated(alias, ty.inner); } } - self.extend(inferred); + self.extend_definition(inferred); } } } @@ -4925,7 +5160,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { standalone_expression: Expression<'db>, ) -> Type<'db> { let types = infer_expression_types(self.db(), standalone_expression); - self.extend(types); + self.extend_expression(types); // Instead of calling `self.expression_type(expr)` after extending here, we get // the result from `types` directly because we might be in cycle recovery where @@ -4990,7 +5225,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // on the string expression itself that represents the annotation. return; } - let previous = self.types.expressions.insert(expression.into(), ty); + let previous = self.expressions.insert(expression.into(), ty); assert_eq!(previous, None); } @@ -5397,10 +5632,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if comprehension.is_first() && target.is_name_expr() { result.expression_type(iterable) } else { - let scope = self.types.scope; - self.types.scope = result.scope; - self.extend(result); - self.types.scope = scope; + self.extend_expression_unchecked(result); result.expression_type(iterable) } }; @@ -5429,7 +5661,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } }; - self.types.expressions.insert(target.into(), target_type); + self.expressions.insert(target.into(), target_type); self.add_binding(target.into(), definition, target_type); } @@ -5438,7 +5670,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if named.target.is_name_expr() { let definition = self.index.expect_single_definition(named); let result = infer_definition_types(self.db(), definition); - self.extend(result); + self.extend_definition(result); result.binding_type(definition) } else { // For syntactically invalid targets, we still need to run type inference: @@ -8515,11 +8747,160 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } - pub(super) fn finish(mut self) -> TypeInference<'db> { + pub(super) fn finish_expression(mut self) -> ExpressionInference<'db> { self.infer_region(); - self.types.diagnostics = self.context.finish(); - self.types.shrink_to_fit(); - self.types + + let Self { + context, + mut expressions, + scope, + bindings, + declarations, + deferred, + cycle_fallback, + + // builder only state + deferred_state: _, + called_functions: _, + index: _, + region: _, + return_types_and_ranges: _, + } = self; + + let diagnostics = context.finish(); + let _ = scope; + + assert!( + declarations.is_empty(), + "Expression region can't have declarations" + ); + assert!( + deferred.is_empty(), + "Expression region can't have deferred types" + ); + + let extra = + (cycle_fallback || !bindings.is_empty() || !diagnostics.is_empty()).then(|| { + if bindings.len() > 20 { + tracing::debug!( + "Inferred expression region `{:?}` contains {} bindings. Lookups by linear scan might be slow.", + self.region, + bindings.len() + ); + } + + Box::new(ExpressionInferenceExtra { + bindings: bindings.into_boxed_slice(), + diagnostics, + cycle_fallback, + }) + }); + + expressions.shrink_to_fit(); + + ExpressionInference { + expressions, + extra, + #[cfg(debug_assertions)] + scope, + } + } + + pub(super) fn finish_definition(mut self) -> DefinitionInference<'db> { + self.infer_region(); + + let Self { + context, + mut expressions, + scope, + bindings, + declarations, + deferred, + cycle_fallback, + + // builder only state + deferred_state: _, + called_functions: _, + index: _, + region: _, + return_types_and_ranges: _, + } = self; + + let _ = scope; + let diagnostics = context.finish(); + + let extra = + (!diagnostics.is_empty() || cycle_fallback || !deferred.is_empty()).then(|| { + Box::new(DefinitionInferenceExtra { + cycle_fallback, + deferred: deferred.into_boxed_slice(), + diagnostics, + }) + }); + + if bindings.len() > 20 { + tracing::debug!( + "Inferred definition region `{:?}` contains {} bindings. Lookups by linear scan might be slow.", + self.region, + bindings.len(), + ); + } + + if declarations.len() > 20 { + tracing::debug!( + "Inferred declaration region `{:?}` contains {} declarations. Lookups by linear scan might be slow.", + self.region, + declarations.len(), + ); + } + + expressions.shrink_to_fit(); + + DefinitionInference { + expressions, + #[cfg(debug_assertions)] + scope, + bindings: bindings.into_boxed_slice(), + declarations: declarations.into_boxed_slice(), + extra, + } + } + + pub(super) fn finish_scope(mut self) -> ScopeInference<'db> { + self.infer_region(); + + let Self { + context, + mut expressions, + scope, + cycle_fallback, + + // Ignored, because scope types are never extended into other scopes. + deferred: _, + bindings: _, + declarations: _, + + // Builder only state + deferred_state: _, + called_functions: _, + index: _, + region: _, + return_types_and_ranges: _, + } = self; + + let _ = scope; + let diagnostics = context.finish(); + + let extra = (!diagnostics.is_empty() || cycle_fallback).then(|| { + Box::new(ScopeInferenceExtra { + cycle_fallback, + diagnostics, + }) + }); + + expressions.shrink_to_fit(); + + ScopeInference { expressions, extra } } } @@ -10340,6 +10721,138 @@ fn contains_string_literal(expr: &ast::Expr) -> bool { visitor.0 } +/// Map based on a `Vec`. It doesn't enforce +/// uniqueness on insertion. Instead, it relies on the caller +/// that elements are uniuqe. For example, the way we visit definitions +/// in the `TypeInference` builder make already implicitly guarantees that each definition +/// is only visited once. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct VecMap(Vec<(K, V)>); + +impl VecMap +where + K: Eq, + K: std::fmt::Debug, + V: std::fmt::Debug, +{ + #[inline] + fn len(&self) -> usize { + self.0.len() + } + + #[inline] + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn iter(&self) -> impl ExactSizeIterator { + self.0.iter().map(|(k, v)| (k, v)) + } + + fn insert(&mut self, key: K, value: V) { + debug_assert!( + !self.0.iter().any(|(existing, _)| existing == &key), + "An existing entry already exists for key {key:?}", + ); + + self.0.push((key, value)); + } + + fn into_boxed_slice(self) -> Box<[(K, V)]> { + self.0.into_boxed_slice() + } +} + +impl Extend<(K, V)> for VecMap +where + K: Eq, + K: std::fmt::Debug, + V: std::fmt::Debug, +{ + #[inline] + fn extend>(&mut self, iter: T) { + if cfg!(debug_assertions) { + for (key, value) in iter { + self.insert(key, value); + } + } else { + self.0.extend(iter); + } + } +} + +impl Default for VecMap { + fn default() -> Self { + Self(Vec::default()) + } +} + +/// Set based on a `Vec`. It doesn't enforce +/// uniqueness on insertion. Instead, it relies on the caller +/// that elements are uniuqe. For example, the way we visit definitions +/// in the `TypeInference` builder make already implicitly guarantees that each definition +/// is only visited once. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct VecSet(Vec); + +impl VecSet { + #[inline] + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn into_boxed_slice(self) -> Box<[V]> { + self.0.into_boxed_slice() + } +} + +impl VecSet +where + V: Eq, + V: std::fmt::Debug, +{ + fn insert(&mut self, value: V) { + debug_assert!( + !self.0.iter().any(|existing| existing == &value), + "An existing entry already exists for {value:?}", + ); + + self.0.push(value); + } +} + +impl Extend for VecSet +where + V: Eq, + V: std::fmt::Debug, +{ + #[inline] + fn extend>(&mut self, iter: T) { + if cfg!(debug_assertions) { + for value in iter { + self.insert(value); + } + } else { + self.0.extend(iter); + } + } +} + +impl Default for VecSet { + fn default() -> Self { + Self(Vec::default()) + } +} + +impl IntoIterator for VecSet { + type Item = V; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + #[cfg(test)] mod tests { use crate::db::tests::{TestDb, setup_db};