language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-30 13:51:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

[red-knot] Better handling of visibility constraint copies (#16276)

Browse source 
Two related changes.  For context:

1. We were maintaining two separate arenas of `Constraint`s in each
use-def map. One was used for narrowing constraints, and the other for
visibility constraints. The visibility constraint arena was interned,
ensuring that we always used the same ID for any particular
`Constraint`. The narrowing constraint arena was not interned.

2. The TDD code relies on _all_ TDD nodes being interned and reduced.
This is an important requirement for TDDs to be a canonical form, which
allows us to use a single int comparison to test for "always true/false"
and to compare two TDDs for equivalence. But we also need to support an
individual `Constraint` having multiple values in a TDD evaluation (e.g.
to handle a `while` condition having different values the first time
it's evaluated vs later times). Previously, we handled that by
introducing a "copy" number, which was only there as a disambiguator, to
allow an interned, deduplicated constraint ID to appear in the TDD
formula multiple times.

A better way to handle (2) is to not intern the constraints in the
visibility constraint arena! The caller now gets to decide: if they add
a `Constraint` to the arena more than once, they get distinct
`ScopedConstraintId`s — which the TDD code will treat as distinct
variables, allowing them to take on different values in the ternary
function.

With that in place, we can then consolidate on a single (non-interned)
arena, which is shared for both narrowing and visibility constraints.

---------

Co-authored-by: Carl Meyer <carl@astral.sh>
This commit is contained in:
Douglas Creager 2025-02-21 09:16:25 -05:00 • committed by GitHub
parent b9b094869a
commit 4dae09ecff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 113 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

31
crates/red_knot_python_semantic/src/semantic_index/builder.rs
View file

@ -15,7 +15,7 @@ use crate::module_name::ModuleName;
use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey; use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey;
use crate::semantic_index::ast_ids::AstIdsBuilder; use crate::semantic_index::ast_ids::AstIdsBuilder;
use crate::semantic_index::attribute_assignment::{AttributeAssignment, AttributeAssignments}; use crate::semantic_index::attribute_assignment::{AttributeAssignment, AttributeAssignments};
use crate::semantic_index::constraint::PatternConstraintKind; use crate::semantic_index::constraint::{PatternConstraintKind, ScopedConstraintId};
use crate::semantic_index::definition::{ use crate::semantic_index::definition::{
AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey, AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey,
DefinitionNodeRef, ForStmtDefinitionNodeRef, ImportFromDefinitionNodeRef, DefinitionNodeRef, ForStmtDefinitionNodeRef, ImportFromDefinitionNodeRef,
@ -26,7 +26,7 @@ use crate::semantic_index::symbol::{
SymbolTableBuilder, SymbolTableBuilder,
}; };
use crate::semantic_index::use_def::{ use crate::semantic_index::use_def::{
EagerBindingsKey, FlowSnapshot, ScopedConstraintId, ScopedEagerBindingsId, UseDefMapBuilder, EagerBindingsKey, FlowSnapshot, ScopedEagerBindingsId, UseDefMapBuilder,
}; };
use crate::semantic_index::SemanticIndex; use crate::semantic_index::SemanticIndex;
use crate::unpack::{Unpack, UnpackValue}; use crate::unpack::{Unpack, UnpackValue};
@ -294,7 +294,7 @@ impl<'db> SemanticIndexBuilder<'db> {
&self.use_def_maps[scope_id] &self.use_def_maps[scope_id]
} }
fn current_visibility_constraints_mut(&mut self) -> &mut VisibilityConstraintsBuilder<'db> { fn current_visibility_constraints_mut(&mut self) -> &mut VisibilityConstraintsBuilder {
let scope_id = self.current_scope(); let scope_id = self.current_scope();
&mut self.use_def_maps[scope_id].visibility_constraints &mut self.use_def_maps[scope_id].visibility_constraints
} }
@ -406,16 +406,12 @@ impl<'db> SemanticIndexBuilder<'db> {
} }
/// Negates a constraint and adds it to the list of all constraints, does not record it. /// Negates a constraint and adds it to the list of all constraints, does not record it.
fn add_negated_constraint( fn add_negated_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
&mut self,
constraint: Constraint<'db>,
) -> (Constraint<'db>, ScopedConstraintId) {
let negated = Constraint { let negated = Constraint {
node: constraint.node, node: constraint.node,
is_positive: false, is_positive: false,
}; };
let id = self.current_use_def_map_mut().add_constraint(negated); self.current_use_def_map_mut().add_constraint(negated)
(negated, id)
} }
/// Records a previously added constraint by adding it to all live bindings. /// Records a previously added constraint by adding it to all live bindings.
@ -431,7 +427,7 @@ impl<'db> SemanticIndexBuilder<'db> {
/// Negates the given constraint and then adds it to all live bindings. /// Negates the given constraint and then adds it to all live bindings.
fn record_negated_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId { fn record_negated_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
let (_, id) = self.add_negated_constraint(constraint); let id = self.add_negated_constraint(constraint);
self.record_constraint_id(id); self.record_constraint_id(id);
id id
} }
@ -460,9 +456,10 @@ impl<'db> SemanticIndexBuilder<'db> {
&mut self, &mut self,
constraint: Constraint<'db>, constraint: Constraint<'db>,
) -> ScopedVisibilityConstraintId { ) -> ScopedVisibilityConstraintId {
let constraint_id = self.current_use_def_map_mut().add_constraint(constraint);
let id = self let id = self
.current_visibility_constraints_mut() .current_visibility_constraints_mut()
.add_atom(constraint, 0); .add_atom(constraint_id);
self.record_visibility_constraint_id(id); self.record_visibility_constraint_id(id);
id id
} }
@ -1192,12 +1189,14 @@ where
// We need multiple copies of the visibility constraint for the while condition, // We need multiple copies of the visibility constraint for the while condition,
// since we need to model situations where the first evaluation of the condition // since we need to model situations where the first evaluation of the condition
// returns True, but a later evaluation returns False. // returns True, but a later evaluation returns False.
let first_constraint_id = self.current_use_def_map_mut().add_constraint(constraint);
let later_constraint_id = self.current_use_def_map_mut().add_constraint(constraint);
let first_vis_constraint_id = self let first_vis_constraint_id = self
.current_visibility_constraints_mut() .current_visibility_constraints_mut()
.add_atom(constraint, 0); .add_atom(first_constraint_id);
let later_vis_constraint_id = self let later_vis_constraint_id = self
.current_visibility_constraints_mut() .current_visibility_constraints_mut()
.add_atom(constraint, 1); .add_atom(later_constraint_id);
// Save aside any break states from an outer loop // Save aside any break states from an outer loop
let saved_break_states = std::mem::take(&mut self.loop_break_states); let saved_break_states = std::mem::take(&mut self.loop_break_states);
@ -1778,13 +1777,13 @@ where
// anymore. // anymore.
if index < values.len() - 1 { if index < values.len() - 1 {
let constraint = self.build_constraint(value); let constraint = self.build_constraint(value);
let (constraint, constraint_id) = match op { let constraint_id = match op {
ast::BoolOp::And => (constraint, self.add_constraint(constraint)), ast::BoolOp::And => self.add_constraint(constraint),
ast::BoolOp::Or => self.add_negated_constraint(constraint), ast::BoolOp::Or => self.add_negated_constraint(constraint),
}; };
let visibility_constraint = self let visibility_constraint = self
.current_visibility_constraints_mut() .current_visibility_constraints_mut()
.add_atom(constraint, 0); .add_atom(constraint_id);
let after_expr = self.flow_snapshot(); let after_expr = self.flow_snapshot();

30
crates/red_knot_python_semantic/src/semantic_index/constraint.rs
View file

@ -1,10 +1,40 @@
use ruff_db::files::File; use ruff_db::files::File;
use ruff_index::{newtype_index, IndexVec};
use ruff_python_ast::Singleton; use ruff_python_ast::Singleton;
use crate::db::Db; use crate::db::Db;
use crate::semantic_index::expression::Expression; use crate::semantic_index::expression::Expression;
use crate::semantic_index::symbol::{FileScopeId, ScopeId}; use crate::semantic_index::symbol::{FileScopeId, ScopeId};
// A scoped identifier for each `Constraint` in a scope.
#[newtype_index]
#[derive(Ord, PartialOrd)]
pub(crate) struct ScopedConstraintId;
// A collection of constraints. This is currently stored in `UseDefMap`, which means we maintain a
// separate set of constraints for each scope in a file.
pub(crate) type Constraints<'db> = IndexVec<ScopedConstraintId, Constraint<'db>>;
#[derive(Debug, Default)]
pub(crate) struct ConstraintsBuilder<'db> {
constraints: IndexVec<ScopedConstraintId, Constraint<'db>>,
}
impl<'db> ConstraintsBuilder<'db> {
/// Adds a constraint. Note that we do not deduplicate constraints. If you add a `Constraint`
/// more than once, you will get distinct `ScopedConstraintId`s for each one. (This lets you
/// model constraint expressions that might evaluate to different values at different points of
/// execution.)
pub(crate) fn add_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
self.constraints.push(constraint)
}
pub(crate) fn build(mut self) -> Constraints<'db> {
self.constraints.shrink_to_fit();
self.constraints
}
}
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, salsa::Update)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, salsa::Update)]
pub(crate) struct Constraint<'db> { pub(crate) struct Constraint<'db> {
pub(crate) node: ConstraintNode<'db>, pub(crate) node: ConstraintNode<'db>,

55
crates/red_knot_python_semantic/src/semantic_index/use_def.rs
View file

@ -165,7 +165,7 @@
//! don't actually store these "list of visible definitions" as a vector of [`Definition`]. //! don't actually store these "list of visible definitions" as a vector of [`Definition`].
//! Instead, [`SymbolBindings`] and [`SymbolDeclarations`] are structs which use bit-sets to track //! Instead, [`SymbolBindings`] and [`SymbolDeclarations`] are structs which use bit-sets to track
//! definitions (and constraints, in the case of bindings) in terms of [`ScopedDefinitionId`] and //! definitions (and constraints, in the case of bindings) in terms of [`ScopedDefinitionId`] and
//! [`ScopedConstraintId`], which are indices into the `all_definitions` and `all_constraints` //! [`ScopedConstraintId`], which are indices into the `all_definitions` and `constraints`
//! indexvecs in the [`UseDefMap`]. //! indexvecs in the [`UseDefMap`].
//! //!
//! There is another special kind of possible "definition" for a symbol: there might be a path from //! There is another special kind of possible "definition" for a symbol: there might be a path from
@ -255,27 +255,27 @@
//! snapshot, and merging a snapshot into the current state. The logic using these methods lives in //! snapshot, and merging a snapshot into the current state. The logic using these methods lives in
//! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it //! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it
//! visits a `StmtIf` node. //! visits a `StmtIf` node.
pub(crate) use self::symbol_state::ScopedConstraintId;
use ruff_index::{newtype_index, IndexVec};
use rustc_hash::FxHashMap;
use self::symbol_state::{ use self::symbol_state::{
ConstraintIndexIterator, LiveBindingsIterator, LiveDeclaration, LiveDeclarationsIterator, ConstraintIndexIterator, LiveBindingsIterator, LiveDeclaration, LiveDeclarationsIterator,
ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState,
}; };
use crate::semantic_index::ast_ids::ScopedUseId; use crate::semantic_index::ast_ids::ScopedUseId;
use crate::semantic_index::constraint::{
Constraint, Constraints, ConstraintsBuilder, ScopedConstraintId,
};
use crate::semantic_index::definition::Definition; use crate::semantic_index::definition::Definition;
use crate::semantic_index::symbol::{FileScopeId, ScopedSymbolId}; use crate::semantic_index::symbol::{FileScopeId, ScopedSymbolId};
use crate::visibility_constraints::{ use crate::visibility_constraints::{
ScopedVisibilityConstraintId, VisibilityConstraints, VisibilityConstraintsBuilder, ScopedVisibilityConstraintId, VisibilityConstraints, VisibilityConstraintsBuilder,
}; };
use ruff_index::{newtype_index, IndexVec};
use rustc_hash::FxHashMap;
use super::constraint::Constraint;
mod bitset; mod bitset;
mod symbol_state; mod symbol_state;
type AllConstraints<'db> = IndexVec<ScopedConstraintId, Constraint<'db>>;
/// Applicable definitions and constraints for every use of a name. /// Applicable definitions and constraints for every use of a name.
#[derive(Debug, PartialEq, Eq, salsa::Update)] #[derive(Debug, PartialEq, Eq, salsa::Update)]
pub(crate) struct UseDefMap<'db> { pub(crate) struct UseDefMap<'db> {
@ -284,10 +284,10 @@ pub(crate) struct UseDefMap<'db> {
all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>, all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
/// Array of [`Constraint`] in this scope. /// Array of [`Constraint`] in this scope.
all_constraints: AllConstraints<'db>, constraints: Constraints<'db>,
/// Array of visibility constraints in this scope. /// Array of visibility constraints in this scope.
visibility_constraints: VisibilityConstraints<'db>, visibility_constraints: VisibilityConstraints,
/// [`SymbolBindings`] reaching a [`ScopedUseId`]. /// [`SymbolBindings`] reaching a [`ScopedUseId`].
bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>, bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>,
@ -369,7 +369,7 @@ impl<'db> UseDefMap<'db> {
) -> BindingWithConstraintsIterator<'map, 'db> { ) -> BindingWithConstraintsIterator<'map, 'db> {
BindingWithConstraintsIterator { BindingWithConstraintsIterator {
all_definitions: &self.all_definitions, all_definitions: &self.all_definitions,
all_constraints: &self.all_constraints, constraints: &self.constraints,
visibility_constraints: &self.visibility_constraints, visibility_constraints: &self.visibility_constraints,
inner: bindings.iter(), inner: bindings.iter(),
} }
@ -381,6 +381,7 @@ impl<'db> UseDefMap<'db> {
) -> DeclarationsIterator<'map, 'db> { ) -> DeclarationsIterator<'map, 'db> {
DeclarationsIterator { DeclarationsIterator {
all_definitions: &self.all_definitions, all_definitions: &self.all_definitions,
constraints: &self.constraints,
visibility_constraints: &self.visibility_constraints, visibility_constraints: &self.visibility_constraints,
inner: declarations.iter(), inner: declarations.iter(),
} }
@ -414,8 +415,8 @@ type EagerBindings = IndexVec<ScopedEagerBindingsId, SymbolBindings>;
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct BindingWithConstraintsIterator<'map, 'db> { pub(crate) struct BindingWithConstraintsIterator<'map, 'db> {
all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>, all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
all_constraints: &'map AllConstraints<'db>, pub(crate) constraints: &'map Constraints<'db>,
pub(crate) visibility_constraints: &'map VisibilityConstraints<'db>, pub(crate) visibility_constraints: &'map VisibilityConstraints,
inner: LiveBindingsIterator<'map>, inner: LiveBindingsIterator<'map>,
} }
@ -423,14 +424,14 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> {
type Item = BindingWithConstraints<'map, 'db>; type Item = BindingWithConstraints<'map, 'db>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let all_constraints = self.all_constraints; let constraints = self.constraints;
self.inner self.inner
.next() .next()
.map(|live_binding| BindingWithConstraints { .map(|live_binding| BindingWithConstraints {
binding: self.all_definitions[live_binding.binding], binding: self.all_definitions[live_binding.binding],
constraints: ConstraintsIterator { narrowing_constraints: ConstraintsIterator {
all_constraints, constraints,
constraint_ids: live_binding.narrowing_constraints.iter(), constraint_ids: live_binding.narrowing_constraints.iter(),
}, },
visibility_constraint: live_binding.visibility_constraint, visibility_constraint: live_binding.visibility_constraint,
@ -442,12 +443,12 @@ impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {}
pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) struct BindingWithConstraints<'map, 'db> {
pub(crate) binding: Option<Definition<'db>>, pub(crate) binding: Option<Definition<'db>>,
pub(crate) constraints: ConstraintsIterator<'map, 'db>, pub(crate) narrowing_constraints: ConstraintsIterator<'map, 'db>,
pub(crate) visibility_constraint: ScopedVisibilityConstraintId, pub(crate) visibility_constraint: ScopedVisibilityConstraintId,
} }
pub(crate) struct ConstraintsIterator<'map, 'db> { pub(crate) struct ConstraintsIterator<'map, 'db> {
all_constraints: &'map AllConstraints<'db>, constraints: &'map Constraints<'db>,
constraint_ids: ConstraintIndexIterator<'map>, constraint_ids: ConstraintIndexIterator<'map>,
} }
@ -457,7 +458,7 @@ impl<'db> Iterator for ConstraintsIterator<'_, 'db> {
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.constraint_ids self.constraint_ids
.next() .next()
.map(|constraint_id| self.all_constraints[ScopedConstraintId::from_u32(constraint_id)]) .map(|constraint_id| self.constraints[ScopedConstraintId::from_u32(constraint_id)])
} }
} }
@ -465,7 +466,8 @@ impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {}
pub(crate) struct DeclarationsIterator<'map, 'db> { pub(crate) struct DeclarationsIterator<'map, 'db> {
all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>, all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
pub(crate) visibility_constraints: &'map VisibilityConstraints<'db>, pub(crate) constraints: &'map Constraints<'db>,
pub(crate) visibility_constraints: &'map VisibilityConstraints,
inner: LiveDeclarationsIterator<'map>, inner: LiveDeclarationsIterator<'map>,
} }
@ -506,11 +508,11 @@ pub(super) struct UseDefMapBuilder<'db> {
/// Append-only array of [`Definition`]. /// Append-only array of [`Definition`].
all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>, all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
/// Append-only array of [`Constraint`]. /// Builder of constraints.
all_constraints: AllConstraints<'db>, constraints: ConstraintsBuilder<'db>,
/// Builder of visibility constraints. /// Builder of visibility constraints.
pub(super) visibility_constraints: VisibilityConstraintsBuilder<'db>, pub(super) visibility_constraints: VisibilityConstraintsBuilder,
/// A constraint which describes the visibility of the unbound/undeclared state, i.e. /// A constraint which describes the visibility of the unbound/undeclared state, i.e.
/// whether or not the start of the scope is visible. This is important for cases like /// whether or not the start of the scope is visible. This is important for cases like
@ -539,7 +541,7 @@ impl Default for UseDefMapBuilder<'_> {
fn default() -> Self { fn default() -> Self {
Self { Self {
all_definitions: IndexVec::from_iter([None]), all_definitions: IndexVec::from_iter([None]),
all_constraints: IndexVec::new(), constraints: ConstraintsBuilder::default(),
visibility_constraints: VisibilityConstraintsBuilder::default(), visibility_constraints: VisibilityConstraintsBuilder::default(),
scope_start_visibility: ScopedVisibilityConstraintId::ALWAYS_TRUE, scope_start_visibility: ScopedVisibilityConstraintId::ALWAYS_TRUE,
bindings_by_use: IndexVec::new(), bindings_by_use: IndexVec::new(),
@ -572,7 +574,7 @@ impl<'db> UseDefMapBuilder<'db> {
} }
pub(super) fn add_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId { pub(super) fn add_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
self.all_constraints.push(constraint) self.constraints.add_constraint(constraint)
} }
pub(super) fn record_constraint_id(&mut self, constraint: ScopedConstraintId) { pub(super) fn record_constraint_id(&mut self, constraint: ScopedConstraintId) {
@ -752,7 +754,6 @@ impl<'db> UseDefMapBuilder<'db> {
pub(super) fn finish(mut self) -> UseDefMap<'db> { pub(super) fn finish(mut self) -> UseDefMap<'db> {
self.all_definitions.shrink_to_fit(); self.all_definitions.shrink_to_fit();
self.all_constraints.shrink_to_fit();
self.symbol_states.shrink_to_fit(); self.symbol_states.shrink_to_fit();
self.bindings_by_use.shrink_to_fit(); self.bindings_by_use.shrink_to_fit();
self.declarations_by_binding.shrink_to_fit(); self.declarations_by_binding.shrink_to_fit();
@ -761,7 +762,7 @@ impl<'db> UseDefMapBuilder<'db> {
UseDefMap { UseDefMap {
all_definitions: self.all_definitions, all_definitions: self.all_definitions,
all_constraints: self.all_constraints, constraints: self.constraints.build(),
visibility_constraints: self.visibility_constraints.build(), visibility_constraints: self.visibility_constraints.build(),
bindings_by_use: self.bindings_by_use, bindings_by_use: self.bindings_by_use,
public_symbols: self.symbol_states, public_symbols: self.symbol_states,

5
crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs
View file

@ -48,6 +48,7 @@ use itertools::{EitherOrBoth, Itertools};
use ruff_index::newtype_index; use ruff_index::newtype_index;
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use crate::semantic_index::constraint::ScopedConstraintId;
use crate::semantic_index::use_def::bitset::{BitSet, BitSetIterator}; use crate::semantic_index::use_def::bitset::{BitSet, BitSetIterator};
use crate::semantic_index::use_def::VisibilityConstraintsBuilder; use crate::semantic_index::use_def::VisibilityConstraintsBuilder;
use crate::visibility_constraints::ScopedVisibilityConstraintId; use crate::visibility_constraints::ScopedVisibilityConstraintId;
@ -66,10 +67,6 @@ impl ScopedDefinitionId {
pub(super) const UNBOUND: ScopedDefinitionId = ScopedDefinitionId::from_u32(0); pub(super) const UNBOUND: ScopedDefinitionId = ScopedDefinitionId::from_u32(0);
} }
/// A newtype-index for a constraint expression in a particular scope.
#[newtype_index]
pub(crate) struct ScopedConstraintId;
/// Can reference this * 64 total constraints inline; more will fall back to the heap. /// Can reference this * 64 total constraints inline; more will fall back to the heap.
const INLINE_CONSTRAINT_BLOCKS: usize = 2; const INLINE_CONSTRAINT_BLOCKS: usize = 2;

18
crates/red_knot_python_semantic/src/symbol.rs
View file

@ -503,6 +503,7 @@ fn symbol_from_bindings_impl<'db>(
bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>, bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>,
requires_explicit_reexport: RequiresExplicitReExport, requires_explicit_reexport: RequiresExplicitReExport,
) -> Symbol<'db> { ) -> Symbol<'db> {
let constraints = bindings_with_constraints.constraints;
let visibility_constraints = bindings_with_constraints.visibility_constraints; let visibility_constraints = bindings_with_constraints.visibility_constraints;
let mut bindings_with_constraints = bindings_with_constraints.peekable(); let mut bindings_with_constraints = bindings_with_constraints.peekable();
@ -514,9 +515,9 @@ fn symbol_from_bindings_impl<'db>(
Some(BindingWithConstraints { Some(BindingWithConstraints {
binding, binding,
visibility_constraint, visibility_constraint,
constraints: _, narrowing_constraints: _,
}) if binding.map_or(true, is_non_exported) => { }) if binding.map_or(true, is_non_exported) => {
visibility_constraints.evaluate(db, *visibility_constraint) visibility_constraints.evaluate(db, constraints, *visibility_constraint)
} }
_ => Truthiness::AlwaysFalse, _ => Truthiness::AlwaysFalse,
}; };
@ -524,7 +525,7 @@ fn symbol_from_bindings_impl<'db>(
let mut types = bindings_with_constraints.filter_map( let mut types = bindings_with_constraints.filter_map(
|BindingWithConstraints { |BindingWithConstraints {
binding, binding,
constraints, narrowing_constraints,
visibility_constraint, visibility_constraint,
}| { }| {
let binding = binding?; let binding = binding?;
@ -533,13 +534,14 @@ fn symbol_from_bindings_impl<'db>(
return None; return None;
} }
let static_visibility = visibility_constraints.evaluate(db, visibility_constraint); let static_visibility =
visibility_constraints.evaluate(db, constraints, visibility_constraint);
if static_visibility.is_always_false() { if static_visibility.is_always_false() {
return None; return None;
} }
let mut constraint_tys = constraints let mut constraint_tys = narrowing_constraints
.filter_map(|constraint| narrowing_constraint(db, constraint, binding)) .filter_map(|constraint| narrowing_constraint(db, constraint, binding))
.peekable(); .peekable();
@ -590,6 +592,7 @@ fn symbol_from_declarations_impl<'db>(
declarations: DeclarationsIterator<'_, 'db>, declarations: DeclarationsIterator<'_, 'db>,
requires_explicit_reexport: RequiresExplicitReExport, requires_explicit_reexport: RequiresExplicitReExport,
) -> SymbolFromDeclarationsResult<'db> { ) -> SymbolFromDeclarationsResult<'db> {
let constraints = declarations.constraints;
let visibility_constraints = declarations.visibility_constraints; let visibility_constraints = declarations.visibility_constraints;
let mut declarations = declarations.peekable(); let mut declarations = declarations.peekable();
@ -602,7 +605,7 @@ fn symbol_from_declarations_impl<'db>(
declaration, declaration,
visibility_constraint, visibility_constraint,
}) if declaration.map_or(true, is_non_exported) => { }) if declaration.map_or(true, is_non_exported) => {
visibility_constraints.evaluate(db, *visibility_constraint) visibility_constraints.evaluate(db, constraints, *visibility_constraint)
} }
_ => Truthiness::AlwaysFalse, _ => Truthiness::AlwaysFalse,
}; };
@ -618,7 +621,8 @@ fn symbol_from_declarations_impl<'db>(
return None; return None;
} }
let static_visibility = visibility_constraints.evaluate(db, visibility_constraint); let static_visibility =
visibility_constraints.evaluate(db, constraints, visibility_constraint);
if static_visibility.is_always_false() { if static_visibility.is_always_false() {
None None

113
crates/red_knot_python_semantic/src/visibility_constraints.rs
View file

@ -178,7 +178,9 @@ use std::cmp::Ordering;
use ruff_index::{Idx, IndexVec}; use ruff_index::{Idx, IndexVec};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use crate::semantic_index::constraint::{Constraint, ConstraintNode, PatternConstraintKind}; use crate::semantic_index::constraint::{
Constraint, ConstraintNode, Constraints, PatternConstraintKind, ScopedConstraintId,
};
use crate::types::{infer_expression_type, Truthiness}; use crate::types::{infer_expression_type, Truthiness};
use crate::Db; use crate::Db;
@ -231,69 +233,15 @@ impl std::fmt::Debug for ScopedVisibilityConstraintId {
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct InteriorNode { struct InteriorNode {
atom: Atom, /// A "variable" that is evaluated as part of a TDD ternary function. For visibility
/// constraints, this is a `Constraint` that represents some runtime property of the Python
/// code that we are evaluating.
atom: ScopedConstraintId,
if_true: ScopedVisibilityConstraintId, if_true: ScopedVisibilityConstraintId,
if_ambiguous: ScopedVisibilityConstraintId, if_ambiguous: ScopedVisibilityConstraintId,
if_false: ScopedVisibilityConstraintId, if_false: ScopedVisibilityConstraintId,
} }
/// A "variable" that is evaluated as part of a TDD ternary function. For visibility constraints,
/// this is a `Constraint` that represents some runtime property of the Python code that we are
/// evaluating. We intern these constraints in an arena ([`VisibilityConstraints::constraints`]).
/// An atom is then an index into this arena.
///
/// By using a 32-bit index, we would typically allow 4 billion distinct constraints within a
/// scope. However, we sometimes have to model how a `Constraint` can have a different runtime
/// value at different points in the execution of the program. To handle this, we reserve the top
/// byte of an atom to represent a "copy number". This is just an opaque value that allows
/// different `Atom`s to evaluate the same `Constraint`. This yields a maximum of 16 million
/// distinct `Constraint`s in a scope, and 256 possible copies of each of those constraints.
#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct Atom(u32);
impl Atom {
/// Deconstruct an atom into a constraint index and a copy number.
#[inline]
fn into_index_and_copy(self) -> (u32, u8) {
let copy = self.0 >> 24;
let index = self.0 & 0x00ff_ffff;
(index, copy as u8)
}
#[inline]
fn copy_of(mut self, copy: u8) -> Self {
// Clear out the previous copy number
self.0 &= 0x00ff_ffff;
// OR in the new one
self.0 |= u32::from(copy) << 24;
self
}
}
// A custom Debug implementation that prints out the constraint index and copy number as distinct
// fields.
impl std::fmt::Debug for Atom {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (index, copy) = self.into_index_and_copy();
f.debug_tuple("Atom").field(&index).field(&copy).finish()
}
}
impl Idx for Atom {
#[inline]
fn new(value: usize) -> Self {
assert!(value <= 0x00ff_ffff);
#[allow(clippy::cast_possible_truncation)]
Self(value as u32)
}
#[inline]
fn index(self) -> usize {
let (index, _) = self.into_index_and_copy();
index as usize
}
}
impl ScopedVisibilityConstraintId { impl ScopedVisibilityConstraintId {
/// A special ID that is used for an "always true" / "always visible" constraint. /// A special ID that is used for an "always true" / "always visible" constraint.
pub(crate) const ALWAYS_TRUE: ScopedVisibilityConstraintId = pub(crate) const ALWAYS_TRUE: ScopedVisibilityConstraintId =
@ -336,16 +284,13 @@ const SMALLEST_TERMINAL: ScopedVisibilityConstraintId = ALWAYS_FALSE;
/// A collection of visibility constraints. This is currently stored in `UseDefMap`, which means we /// A collection of visibility constraints. This is currently stored in `UseDefMap`, which means we
/// maintain a separate set of visibility constraints for each scope in file. /// maintain a separate set of visibility constraints for each scope in file.
#[derive(Debug, PartialEq, Eq, salsa::Update)] #[derive(Debug, PartialEq, Eq, salsa::Update)]
pub(crate) struct VisibilityConstraints<'db> { pub(crate) struct VisibilityConstraints {
constraints: IndexVec<Atom, Constraint<'db>>,
interiors: IndexVec<ScopedVisibilityConstraintId, InteriorNode>, interiors: IndexVec<ScopedVisibilityConstraintId, InteriorNode>,
} }
#[derive(Debug, Default, PartialEq, Eq)] #[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct VisibilityConstraintsBuilder<'db> { pub(crate) struct VisibilityConstraintsBuilder {
constraints: IndexVec<Atom, Constraint<'db>>,
interiors: IndexVec<ScopedVisibilityConstraintId, InteriorNode>, interiors: IndexVec<ScopedVisibilityConstraintId, InteriorNode>,
constraint_cache: FxHashMap<Constraint<'db>, Atom>,
interior_cache: FxHashMap<InteriorNode, ScopedVisibilityConstraintId>, interior_cache: FxHashMap<InteriorNode, ScopedVisibilityConstraintId>,
not_cache: FxHashMap<ScopedVisibilityConstraintId, ScopedVisibilityConstraintId>, not_cache: FxHashMap<ScopedVisibilityConstraintId, ScopedVisibilityConstraintId>,
and_cache: FxHashMap< and_cache: FxHashMap<
@ -358,10 +303,9 @@ pub(crate) struct VisibilityConstraintsBuilder<'db> {
>, >,
} }
impl<'db> VisibilityConstraintsBuilder<'db> { impl VisibilityConstraintsBuilder {
pub(crate) fn build(self) -> VisibilityConstraints<'db> { pub(crate) fn build(self) -> VisibilityConstraints {
VisibilityConstraints { VisibilityConstraints {
constraints: self.constraints,
interiors: self.interiors, interiors: self.interiors,
} }
} }
@ -385,14 +329,6 @@ impl<'db> VisibilityConstraintsBuilder<'db> {
} }
} }
/// Adds a constraint, ensuring that we only store any particular constraint once.
fn add_constraint(&mut self, constraint: Constraint<'db>, copy: u8) -> Atom {
self.constraint_cache
.entry(constraint)
.or_insert_with(|| self.constraints.push(constraint))
.copy_of(copy)
}
/// Adds an interior node, ensuring that we always use the same visibility constraint ID for /// Adds an interior node, ensuring that we always use the same visibility constraint ID for
/// equal nodes. /// equal nodes.
fn add_interior(&mut self, node: InteriorNode) -> ScopedVisibilityConstraintId { fn add_interior(&mut self, node: InteriorNode) -> ScopedVisibilityConstraintId {
@ -408,17 +344,23 @@ impl<'db> VisibilityConstraintsBuilder<'db> {
.or_insert_with(|| self.interiors.push(node)) .or_insert_with(|| self.interiors.push(node))
} }
/// Adds a new visibility constraint that checks a single [`Constraint`]. Provide different /// Adds a new visibility constraint that checks a single [`Constraint`].
/// values for `copy` if you need to model that the constraint can evaluate to different ///
/// results at different points in the execution of the program being modeled. /// [`ScopedConstraintId`]s are the “variables” that are evaluated by a TDD. A TDD variable has
/// the same value no matter how many times it appears in the ternary formula that the TDD
/// represents.
///
/// However, we sometimes have to model how a `Constraint` can have a different runtime
/// value at different points in the execution of the program. To handle this, you can take
/// advantage of the fact that the [`Constraints`] arena does not deduplicate `Constraint`s.
/// You can add a `Constraint` multiple times, yielding different `ScopedConstraintId`s, which
/// you can then create separate TDD atoms for.
pub(crate) fn add_atom( pub(crate) fn add_atom(
&mut self, &mut self,
constraint: Constraint<'db>, constraint: ScopedConstraintId,
copy: u8,
) -> ScopedVisibilityConstraintId { ) -> ScopedVisibilityConstraintId {
let atom = self.add_constraint(constraint, copy);
self.add_interior(InteriorNode { self.add_interior(InteriorNode {
atom, atom: constraint,
if_true: ALWAYS_TRUE, if_true: ALWAYS_TRUE,
if_ambiguous: AMBIGUOUS, if_ambiguous: AMBIGUOUS,
if_false: ALWAYS_FALSE, if_false: ALWAYS_FALSE,
@ -588,11 +530,12 @@ impl<'db> VisibilityConstraintsBuilder<'db> {
} }
} }
impl<'db> VisibilityConstraints<'db> { impl VisibilityConstraints {
/// Analyze the statically known visibility for a given visibility constraint. /// Analyze the statically known visibility for a given visibility constraint.
pub(crate) fn evaluate( pub(crate) fn evaluate<'db>(
&self, &self,
db: &'db dyn Db, db: &'db dyn Db,
constraints: &Constraints<'db>,
mut id: ScopedVisibilityConstraintId, mut id: ScopedVisibilityConstraintId,
) -> Truthiness { ) -> Truthiness {
loop { loop {
@ -602,7 +545,7 @@ impl<'db> VisibilityConstraints<'db> {
ALWAYS_FALSE => return Truthiness::AlwaysFalse, ALWAYS_FALSE => return Truthiness::AlwaysFalse,
_ => self.interiors[id], _ => self.interiors[id],
}; };
let constraint = &self.constraints[node.atom]; let constraint = &constraints[node.atom];
match Self::analyze_single(db, constraint) { match Self::analyze_single(db, constraint) {
Truthiness::AlwaysTrue => id = node.if_true, Truthiness::AlwaysTrue => id = node.if_true,
Truthiness::Ambiguous => id = node.if_ambiguous, Truthiness::Ambiguous => id = node.if_ambiguous,