[red-knot] Migrate CLI to Salsa (#11972)

Cargo.lock

@@ -503,6 +503,11 @@ name = "countme"
 version = "3.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
+dependencies = [
+ "dashmap 5.5.3",
+ "once_cell",
+ "rustc-hash 1.1.0",
+]
 
 [[package]]
 name = "crc32fast"
@@ -1853,27 +1858,20 @@ dependencies = [
 
 [[package]]
 name = "red_knot"
-version = "0.1.0"
+version = "0.0.0"
 dependencies = [
  "anyhow",
- "bitflags 2.6.0",
+ "countme",
  "crossbeam",
  "ctrlc",
- "dashmap 6.0.1",
- "hashbrown 0.14.5",
- "indexmap",
- "is-macro",
  "notify",
- "parking_lot",
  "rayon",
  "red_knot_module_resolver",
- "ruff_index",
+ "red_knot_python_semantic",
- "ruff_notebook",
+ "ruff_db",
  "ruff_python_ast",
- "ruff_python_parser",
- "ruff_text_size",
  "rustc-hash 2.0.0",
- "tempfile",
+ "salsa",
  "tracing",
  "tracing-subscriber",
  "tracing-tree",

Cargo.toml

@@ -36,6 +36,7 @@ ruff_text_size = { path = "crates/ruff_text_size" }
 ruff_workspace = { path = "crates/ruff_workspace" }
 
 red_knot_module_resolver = { path = "crates/red_knot_module_resolver" }
+red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
 
 aho-corasick = { version = "1.1.3" }
 annotate-snippets = { version = "0.9.2", features = ["color"] }
@@ -96,7 +97,6 @@ once_cell = { version = "1.19.0" }
 path-absolutize = { version = "3.1.1" }
 path-slash = { version = "0.2.1" }
 pathdiff = { version = "0.2.1" }
-parking_lot = "0.12.1"
 pep440_rs = { version = "0.6.0", features = ["serde"] }
 pretty_assertions = "1.3.0"
 proc-macro2 = { version = "1.0.79" }

crates/red_knot/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "red_knot"
-version = "0.1.0"
+version = "0.0.0"
 edition.workspace = true
 rust-version.workspace = true
 homepage.workspace = true
@@ -8,36 +8,29 @@ documentation.workspace = true
 repository.workspace = true
 authors.workspace = true
 license.workspace = true
+default-run = "red_knot"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 red_knot_module_resolver = { workspace = true }
+red_knot_python_semantic = { workspace = true }
 
-ruff_python_parser = { workspace = true }
+ruff_db = { workspace = true }
 ruff_python_ast = { workspace = true }
-ruff_text_size = { workspace = true }
-ruff_index = { workspace = true }
-ruff_notebook = { workspace = true }
 
 anyhow = { workspace = true }
-bitflags = { workspace = true }
+countme = { workspace = true, features = ["enable"] }
 crossbeam = { workspace = true }
 ctrlc = { version = "3.4.4" }
-dashmap = { workspace = true }
-hashbrown = { workspace = true }
-indexmap = { workspace = true }
-is-macro = { workspace = true }
 notify = { workspace = true }
-parking_lot = { workspace = true }
 rayon = { workspace = true }
 rustc-hash = { workspace = true }
+salsa = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing-tree = { workspace = true }
 
-[dev-dependencies]
-tempfile = { workspace = true }
 
 [lints]
 workspace = true

crates/red_knot/src/ast_ids.rs

@@ -1,418 +0,0 @@
-use std::any::type_name;
-use std::fmt::{Debug, Formatter};
-use std::hash::{Hash, Hasher};
-use std::marker::PhantomData;
-
-use rustc_hash::FxHashMap;
-
-use ruff_index::{Idx, IndexVec};
-use ruff_python_ast::visitor::source_order;
-use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal};
-use ruff_python_ast::{
-    AnyNodeRef, AstNode, ExceptHandler, ExceptHandlerExceptHandler, Expr, MatchCase, ModModule,
-    NodeKind, Parameter, Stmt, StmtAnnAssign, StmtAssign, StmtAugAssign, StmtClassDef,
-    StmtFunctionDef, StmtGlobal, StmtImport, StmtImportFrom, StmtNonlocal, StmtTypeAlias,
-    TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, WithItem,
-};
-use ruff_text_size::{Ranged, TextRange};
-
-/// A type agnostic ID that uniquely identifies an AST node in a file.
-#[ruff_index::newtype_index]
-pub struct AstId;
-
-/// A typed ID that uniquely identifies an AST node in a file.
-///
-/// This is different from [`AstId`] in that it is a combination of ID and the type of the node the ID identifies.
-/// Typing the ID prevents mixing IDs of different node types and allows to restrict the API to only accept
-/// nodes for which an ID has been created (not all AST nodes get an ID).
-pub struct TypedAstId<N: HasAstId> {
-    erased: AstId,
-    _marker: PhantomData<fn() -> N>,
-}
-
-impl<N: HasAstId> TypedAstId<N> {
-    /// Upcasts this ID from a more specific node type to a more general node type.
-    pub fn upcast<M: HasAstId>(self) -> TypedAstId<M>
-    where
-        N: Into<M>,
-    {
-        TypedAstId {
-            erased: self.erased,
-            _marker: PhantomData,
-        }
-    }
-}
-
-impl<N: HasAstId> Copy for TypedAstId<N> {}
-impl<N: HasAstId> Clone for TypedAstId<N> {
-    fn clone(&self) -> Self {
-        *self
-    }
-}
-
-impl<N: HasAstId> PartialEq for TypedAstId<N> {
-    fn eq(&self, other: &Self) -> bool {
-        self.erased == other.erased
-    }
-}
-
-impl<N: HasAstId> Eq for TypedAstId<N> {}
-impl<N: HasAstId> Hash for TypedAstId<N> {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.erased.hash(state);
-    }
-}
-
-impl<N: HasAstId> Debug for TypedAstId<N> {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("TypedAstId")
-            .field(&self.erased)
-            .field(&type_name::<N>())
-            .finish()
-    }
-}
-
-pub struct AstIds {
-    ids: IndexVec<AstId, NodeKey>,
-    reverse: FxHashMap<NodeKey, AstId>,
-}
-
-impl AstIds {
-    // TODO rust analyzer doesn't allocate an ID for every node. It only allocates ids for
-    //  nodes with a corresponding HIR element, that is nodes that are definitions.
-    pub fn from_module(module: &ModModule) -> Self {
-        let mut visitor = AstIdsVisitor::default();
-
-        // TODO: visit_module?
-        // Make sure we visit the root
-        visitor.create_id(module);
-        visitor.visit_body(&module.body);
-
-        while let Some(deferred) = visitor.deferred.pop() {
-            match deferred {
-                DeferredNode::FunctionDefinition(def) => {
-                    def.visit_source_order(&mut visitor);
-                }
-                DeferredNode::ClassDefinition(def) => def.visit_source_order(&mut visitor),
-            }
-        }
-
-        AstIds {
-            ids: visitor.ids,
-            reverse: visitor.reverse,
-        }
-    }
-
-    /// Returns the ID to the root node.
-    pub fn root(&self) -> NodeKey {
-        self.ids[AstId::new(0)]
-    }
-
-    /// Returns the [`TypedAstId`] for a node.
-    pub fn ast_id<N: HasAstId>(&self, node: &N) -> TypedAstId<N> {
-        let key = node.syntax_node_key();
-        TypedAstId {
-            erased: self.reverse.get(&key).copied().unwrap(),
-            _marker: PhantomData,
-        }
-    }
-
-    /// Returns the [`TypedAstId`] for the node identified with the given [`TypedNodeKey`].
-    pub fn ast_id_for_key<N: HasAstId>(&self, node: &TypedNodeKey<N>) -> TypedAstId<N> {
-        let ast_id = self.ast_id_for_node_key(node.inner);
-
-        TypedAstId {
-            erased: ast_id,
-            _marker: PhantomData,
-        }
-    }
-
-    /// Returns the untyped [`AstId`] for the node identified by the given `node` key.
-    pub fn ast_id_for_node_key(&self, node: NodeKey) -> AstId {
-        self.reverse
-            .get(&node)
-            .copied()
-            .expect("Can't find node in AstIds map.")
-    }
-
-    /// Returns the [`TypedNodeKey`] for the node identified by the given [`TypedAstId`].
-    pub fn key<N: HasAstId>(&self, id: TypedAstId<N>) -> TypedNodeKey<N> {
-        let syntax_key = self.ids[id.erased];
-
-        TypedNodeKey::new(syntax_key).unwrap()
-    }
-
-    pub fn node_key<H: HasAstId>(&self, id: TypedAstId<H>) -> NodeKey {
-        self.ids[id.erased]
-    }
-}
-
-impl std::fmt::Debug for AstIds {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let mut map = f.debug_map();
-        for (key, value) in self.ids.iter_enumerated() {
-            map.entry(&key, &value);
-        }
-
-        map.finish()
-    }
-}
-
-impl PartialEq for AstIds {
-    fn eq(&self, other: &Self) -> bool {
-        self.ids == other.ids
-    }
-}
-
-impl Eq for AstIds {}
-
-#[derive(Default)]
-struct AstIdsVisitor<'a> {
-    ids: IndexVec<AstId, NodeKey>,
-    reverse: FxHashMap<NodeKey, AstId>,
-    deferred: Vec<DeferredNode<'a>>,
-}
-
-impl<'a> AstIdsVisitor<'a> {
-    fn create_id<A: HasAstId>(&mut self, node: &A) {
-        let node_key = node.syntax_node_key();
-
-        let id = self.ids.push(node_key);
-        self.reverse.insert(node_key, id);
-    }
-}
-
-impl<'a> SourceOrderVisitor<'a> for AstIdsVisitor<'a> {
-    fn visit_stmt(&mut self, stmt: &'a Stmt) {
-        match stmt {
-            Stmt::FunctionDef(def) => {
-                self.create_id(def);
-                self.deferred.push(DeferredNode::FunctionDefinition(def));
-                return;
-            }
-            // TODO defer visiting the assignment body, type alias parameters etc?
-            Stmt::ClassDef(def) => {
-                self.create_id(def);
-                self.deferred.push(DeferredNode::ClassDefinition(def));
-                return;
-            }
-            Stmt::Expr(_) => {
-                // Skip
-                return;
-            }
-            Stmt::Return(_) => {}
-            Stmt::Delete(_) => {}
-            Stmt::Assign(assignment) => self.create_id(assignment),
-            Stmt::AugAssign(assignment) => {
-                self.create_id(assignment);
-            }
-            Stmt::AnnAssign(assignment) => self.create_id(assignment),
-            Stmt::TypeAlias(assignment) => self.create_id(assignment),
-            Stmt::For(_) => {}
-            Stmt::While(_) => {}
-            Stmt::If(_) => {}
-            Stmt::With(_) => {}
-            Stmt::Match(_) => {}
-            Stmt::Raise(_) => {}
-            Stmt::Try(_) => {}
-            Stmt::Assert(_) => {}
-            Stmt::Import(import) => self.create_id(import),
-            Stmt::ImportFrom(import_from) => self.create_id(import_from),
-            Stmt::Global(global) => self.create_id(global),
-            Stmt::Nonlocal(non_local) => self.create_id(non_local),
-            Stmt::Pass(_) => {}
-            Stmt::Break(_) => {}
-            Stmt::Continue(_) => {}
-            Stmt::IpyEscapeCommand(_) => {}
-        }
-
-        source_order::walk_stmt(self, stmt);
-    }
-
-    fn visit_expr(&mut self, _expr: &'a Expr) {}
-
-    fn visit_parameter(&mut self, parameter: &'a Parameter) {
-        self.create_id(parameter);
-        source_order::walk_parameter(self, parameter);
-    }
-
-    fn visit_except_handler(&mut self, except_handler: &'a ExceptHandler) {
-        match except_handler {
-            ExceptHandler::ExceptHandler(except_handler) => {
-                self.create_id(except_handler);
-            }
-        }
-
-        source_order::walk_except_handler(self, except_handler);
-    }
-
-    fn visit_with_item(&mut self, with_item: &'a WithItem) {
-        self.create_id(with_item);
-        source_order::walk_with_item(self, with_item);
-    }
-
-    fn visit_match_case(&mut self, match_case: &'a MatchCase) {
-        self.create_id(match_case);
-        source_order::walk_match_case(self, match_case);
-    }
-
-    fn visit_type_param(&mut self, type_param: &'a TypeParam) {
-        self.create_id(type_param);
-    }
-}
-
-enum DeferredNode<'a> {
-    FunctionDefinition(&'a StmtFunctionDef),
-    ClassDefinition(&'a StmtClassDef),
-}
-
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
-pub struct TypedNodeKey<N: AstNode> {
-    /// The type erased node key.
-    inner: NodeKey,
-    _marker: PhantomData<fn() -> N>,
-}
-
-impl<N: AstNode> TypedNodeKey<N> {
-    pub fn from_node(node: &N) -> Self {
-        let inner = NodeKey::from_node(node.as_any_node_ref());
-        Self {
-            inner,
-            _marker: PhantomData,
-        }
-    }
-
-    pub fn new(node_key: NodeKey) -> Option<Self> {
-        N::can_cast(node_key.kind).then_some(TypedNodeKey {
-            inner: node_key,
-            _marker: PhantomData,
-        })
-    }
-
-    pub fn resolve<'a>(&self, root: AnyNodeRef<'a>) -> Option<N::Ref<'a>> {
-        let node_ref = self.inner.resolve(root)?;
-
-        Some(N::cast_ref(node_ref).unwrap())
-    }
-
-    pub fn resolve_unwrap<'a>(&self, root: AnyNodeRef<'a>) -> N::Ref<'a> {
-        self.resolve(root).expect("node should resolve")
-    }
-
-    pub fn erased(&self) -> &NodeKey {
-        &self.inner
-    }
-}
-
-struct FindNodeKeyVisitor<'a> {
-    key: NodeKey,
-    result: Option<AnyNodeRef<'a>>,
-}
-
-impl<'a> SourceOrderVisitor<'a> for FindNodeKeyVisitor<'a> {
-    fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal {
-        if self.result.is_some() {
-            return TraversalSignal::Skip;
-        }
-
-        if node.range() == self.key.range && node.kind() == self.key.kind {
-            self.result = Some(node);
-            TraversalSignal::Skip
-        } else if node.range().contains_range(self.key.range) {
-            TraversalSignal::Traverse
-        } else {
-            TraversalSignal::Skip
-        }
-    }
-
-    fn visit_body(&mut self, body: &'a [Stmt]) {
-        // TODO it would be more efficient to use binary search instead of linear
-        for stmt in body {
-            if stmt.range().start() > self.key.range.end() {
-                break;
-            }
-
-            self.visit_stmt(stmt);
-        }
-    }
-}
-
-// TODO an alternative to this is to have a `NodeId` on each node (in increasing order depending on the position).
-//  This would allow to reduce the size of this to a u32.
-//  What would be nice if we could use an `Arc::weak_ref` here but that only works if we use
-//   `Arc` internally
-// TODO: Implement the logic to resolve a node, given a db (and the correct file).
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
-pub struct NodeKey {
-    kind: NodeKind,
-    range: TextRange,
-}
-
-impl NodeKey {
-    pub fn from_node(node: AnyNodeRef) -> Self {
-        NodeKey {
-            kind: node.kind(),
-            range: node.range(),
-        }
-    }
-    pub fn resolve<'a>(&self, root: AnyNodeRef<'a>) -> Option<AnyNodeRef<'a>> {
-        // We need to do a binary search here. Only traverse into a node if the range is withint the node
-        let mut visitor = FindNodeKeyVisitor {
-            key: *self,
-            result: None,
-        };
-
-        if visitor.enter_node(root) == TraversalSignal::Traverse {
-            root.visit_preorder(&mut visitor);
-        }
-
-        visitor.result
-    }
-}
-
-/// Marker trait implemented by AST nodes for which we extract the `AstId`.
-pub trait HasAstId: AstNode {
-    fn node_key(&self) -> TypedNodeKey<Self>
-    where
-        Self: Sized,
-    {
-        TypedNodeKey {
-            inner: self.syntax_node_key(),
-            _marker: PhantomData,
-        }
-    }
-
-    fn syntax_node_key(&self) -> NodeKey {
-        NodeKey {
-            kind: self.as_any_node_ref().kind(),
-            range: self.range(),
-        }
-    }
-}
-
-impl HasAstId for StmtFunctionDef {}
-impl HasAstId for StmtClassDef {}
-impl HasAstId for StmtAnnAssign {}
-impl HasAstId for StmtAugAssign {}
-impl HasAstId for StmtAssign {}
-impl HasAstId for StmtTypeAlias {}
-
-impl HasAstId for ModModule {}
-
-impl HasAstId for StmtImport {}
-
-impl HasAstId for StmtImportFrom {}
-
-impl HasAstId for Parameter {}
-
-impl HasAstId for TypeParam {}
-impl HasAstId for Stmt {}
-impl HasAstId for TypeParamTypeVar {}
-impl HasAstId for TypeParamTypeVarTuple {}
-impl HasAstId for TypeParamParamSpec {}
-impl HasAstId for StmtGlobal {}
-impl HasAstId for StmtNonlocal {}
-
-impl HasAstId for ExceptHandlerExceptHandler {}
-impl HasAstId for WithItem {}
-impl HasAstId for MatchCase {}

crates/red_knot/src/cache.rs

@@ -1,165 +0,0 @@
-use std::fmt::Formatter;
-use std::hash::Hash;
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-use crate::db::QueryResult;
-use dashmap::mapref::entry::Entry;
-
-use crate::FxDashMap;
-
-/// Simple key value cache that locks on a per-key level.
-pub struct KeyValueCache<K, V> {
-    map: FxDashMap<K, V>,
-    statistics: CacheStatistics,
-}
-
-impl<K, V> KeyValueCache<K, V>
-where
-    K: Eq + Hash + Clone,
-    V: Clone,
-{
-    pub fn try_get(&self, key: &K) -> Option<V> {
-        if let Some(existing) = self.map.get(key) {
-            self.statistics.hit();
-            Some(existing.clone())
-        } else {
-            self.statistics.miss();
-            None
-        }
-    }
-
-    pub fn get<F>(&self, key: &K, compute: F) -> QueryResult<V>
-    where
-        F: FnOnce(&K) -> QueryResult<V>,
-    {
-        Ok(match self.map.entry(key.clone()) {
-            Entry::Occupied(cached) => {
-                self.statistics.hit();
-
-                cached.get().clone()
-            }
-            Entry::Vacant(vacant) => {
-                self.statistics.miss();
-
-                let value = compute(key)?;
-                vacant.insert(value.clone());
-                value
-            }
-        })
-    }
-
-    pub fn set(&mut self, key: K, value: V) {
-        self.map.insert(key, value);
-    }
-
-    pub fn remove(&mut self, key: &K) -> Option<V> {
-        self.map.remove(key).map(|(_, value)| value)
-    }
-
-    pub fn clear(&mut self) {
-        self.map.clear();
-        self.map.shrink_to_fit();
-    }
-
-    pub fn statistics(&self) -> Option<Statistics> {
-        self.statistics.to_statistics()
-    }
-}
-
-impl<K, V> Default for KeyValueCache<K, V>
-where
-    K: Eq + Hash,
-    V: Clone,
-{
-    fn default() -> Self {
-        Self {
-            map: FxDashMap::default(),
-            statistics: CacheStatistics::default(),
-        }
-    }
-}
-
-impl<K, V> std::fmt::Debug for KeyValueCache<K, V>
-where
-    K: std::fmt::Debug + Eq + Hash,
-    V: std::fmt::Debug,
-{
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let mut debug = f.debug_map();
-
-        for entry in &self.map {
-            debug.entry(&entry.value(), &entry.key());
-        }
-
-        debug.finish()
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Statistics {
-    pub hits: usize,
-    pub misses: usize,
-}
-
-impl Statistics {
-    #[allow(clippy::cast_precision_loss)]
-    pub fn hit_rate(&self) -> Option<f64> {
-        if self.hits + self.misses == 0 {
-            return None;
-        }
-
-        Some((self.hits as f64) / (self.hits + self.misses) as f64)
-    }
-}
-
-#[cfg(debug_assertions)]
-pub type CacheStatistics = DebugStatistics;
-
-#[cfg(not(debug_assertions))]
-pub type CacheStatistics = ReleaseStatistics;
-
-pub trait StatisticsRecorder {
-    fn hit(&self);
-    fn miss(&self);
-    fn to_statistics(&self) -> Option<Statistics>;
-}
-
-#[derive(Debug, Default)]
-pub struct DebugStatistics {
-    hits: AtomicUsize,
-    misses: AtomicUsize,
-}
-
-impl StatisticsRecorder for DebugStatistics {
-    // TODO figure out appropriate Ordering
-    fn hit(&self) {
-        self.hits.fetch_add(1, Ordering::SeqCst);
-    }
-
-    fn miss(&self) {
-        self.misses.fetch_add(1, Ordering::SeqCst);
-    }
-
-    fn to_statistics(&self) -> Option<Statistics> {
-        let hits = self.hits.load(Ordering::SeqCst);
-        let misses = self.misses.load(Ordering::SeqCst);
-
-        Some(Statistics { hits, misses })
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct ReleaseStatistics;
-
-impl StatisticsRecorder for ReleaseStatistics {
-    #[inline]
-    fn hit(&self) {}
-
-    #[inline]
-    fn miss(&self) {}
-
-    #[inline]
-    fn to_statistics(&self) -> Option<Statistics> {
-        None
-    }
-}

crates/red_knot/src/cancellation.rs

@@ -1,42 +0,0 @@
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
-
-#[derive(Debug, Clone, Default)]
-pub struct CancellationTokenSource {
-    signal: Arc<AtomicBool>,
-}
-
-impl CancellationTokenSource {
-    pub fn new() -> Self {
-        Self {
-            signal: Arc::new(AtomicBool::new(false)),
-        }
-    }
-
-    #[tracing::instrument(level = "trace", skip_all)]
-    pub fn cancel(&self) {
-        self.signal.store(true, std::sync::atomic::Ordering::SeqCst);
-    }
-
-    pub fn is_cancelled(&self) -> bool {
-        self.signal.load(std::sync::atomic::Ordering::SeqCst)
-    }
-
-    pub fn token(&self) -> CancellationToken {
-        CancellationToken {
-            signal: self.signal.clone(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct CancellationToken {
-    signal: Arc<AtomicBool>,
-}
-
-impl CancellationToken {
-    /// Returns `true` if cancellation has been requested.
-    pub fn is_cancelled(&self) -> bool {
-        self.signal.load(std::sync::atomic::Ordering::SeqCst)
-    }
-}

crates/red_knot/src/db.rs

@@ -1,248 +1,10 @@
-use std::sync::Arc;
+use red_knot_python_semantic::Db as SemanticDb;
+use ruff_db::Upcast;
+use salsa::DbWithJar;
 
-pub use jars::{HasJar, HasJars};
+use crate::lint::{lint_semantic, lint_syntax, unwind_if_cancelled};
-pub use query::{QueryError, QueryResult};
-pub use runtime::DbRuntime;
-pub use storage::JarsStorage;
 
-use crate::files::FileId;
+pub trait Db: DbWithJar<Jar> + SemanticDb + Upcast<dyn SemanticDb> {}
-use crate::lint::{LintSemanticStorage, LintSyntaxStorage};
-use crate::module::ModuleResolver;
-use crate::parse::ParsedStorage;
-use crate::semantic::SemanticIndexStorage;
-use crate::semantic::TypeStore;
-use crate::source::SourceStorage;
 
-mod jars;
+#[salsa::jar(db=Db)]
-mod query;
+pub struct Jar(lint_syntax, lint_semantic, unwind_if_cancelled);
-mod runtime;
-mod storage;
-
-pub trait Database {
-    /// Returns a reference to the runtime of the current worker.
-    fn runtime(&self) -> &DbRuntime;
-
-    /// Returns a mutable reference to the runtime. Only one worker can hold a mutable reference to the runtime.
-    fn runtime_mut(&mut self) -> &mut DbRuntime;
-
-    /// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
-    fn cancelled(&self) -> QueryResult<()> {
-        self.runtime().cancelled()
-    }
-
-    /// Returns `true` if the queries have been cancelled.
-    fn is_cancelled(&self) -> bool {
-        self.runtime().is_cancelled()
-    }
-}
-
-/// Database that supports running queries from multiple threads.
-pub trait ParallelDatabase: Database + Send {
-    /// Creates a snapshot of the database state that can be used to query the database in another thread.
-    ///
-    /// The snapshot is a read-only view of the database but query results are shared between threads.
-    /// All queries will be automatically cancelled when applying any mutations (calling [`HasJars::jars_mut`])
-    /// to the database (not the snapshot, because they're readonly).
-    ///
-    /// ## Creating a snapshot
-    ///
-    /// Creating a snapshot of the database's jars is cheap but creating a snapshot of
-    /// other state stored on the database might require deep-cloning data. That's why you should
-    /// avoid creating snapshots in a hot function (e.g. don't create a snapshot for each file, instead
-    /// create a snapshot when scheduling the check of an entire program).
-    ///
-    /// ## Salsa compatibility
-    /// Salsa prohibits creating a snapshot while running a local query (it's fine if other workers run a query) [[source](https://github.com/salsa-rs/salsa/issues/80)].
-    /// We should avoid creating snapshots while running a query because we might want to adopt Salsa in the future (if we can figure out persistent caching).
-    /// Unfortunately, the infrastructure doesn't provide an automated way of knowing when a query is run, that's
-    /// why we have to "enforce" this constraint manually.
-    #[must_use]
-    fn snapshot(&self) -> Snapshot<Self>;
-}
-
-pub trait DbWithJar<Jar>: Database + HasJar<Jar> {}
-
-/// Readonly snapshot of a database.
-///
-/// ## Dead locks
-/// A snapshot should always be dropped as soon as it is no longer necessary to run queries.
-/// Storing the snapshot without running a query or periodically checking if cancellation was requested
-/// can lead to deadlocks because mutating the [`Database`] requires cancels all pending queries
-/// and waiting for all [`Snapshot`]s to be dropped.
-#[derive(Debug)]
-pub struct Snapshot<DB: ?Sized>
-where
-    DB: ParallelDatabase,
-{
-    db: DB,
-}
-
-impl<DB> Snapshot<DB>
-where
-    DB: ParallelDatabase,
-{
-    pub fn new(db: DB) -> Self {
-        Snapshot { db }
-    }
-}
-
-impl<DB> std::ops::Deref for Snapshot<DB>
-where
-    DB: ParallelDatabase,
-{
-    type Target = DB;
-
-    fn deref(&self) -> &DB {
-        &self.db
-    }
-}
-
-pub trait Upcast<T: ?Sized> {
-    fn upcast(&self) -> &T;
-}
-
-// Red knot specific databases code.
-
-pub trait SourceDb: DbWithJar<SourceJar> {
-    // queries
-    fn file_id(&self, path: &std::path::Path) -> FileId;
-
-    fn file_path(&self, file_id: FileId) -> Arc<std::path::Path>;
-}
-
-pub trait SemanticDb: SourceDb + DbWithJar<SemanticJar> + Upcast<dyn SourceDb> {}
-
-pub trait LintDb: SemanticDb + DbWithJar<LintJar> + Upcast<dyn SemanticDb> {}
-
-pub trait Db: LintDb + Upcast<dyn LintDb> {}
-
-#[derive(Debug, Default)]
-pub struct SourceJar {
-    pub sources: SourceStorage,
-    pub parsed: ParsedStorage,
-}
-
-#[derive(Debug, Default)]
-pub struct SemanticJar {
-    pub module_resolver: ModuleResolver,
-    pub semantic_indices: SemanticIndexStorage,
-    pub type_store: TypeStore,
-}
-
-#[derive(Debug, Default)]
-pub struct LintJar {
-    pub lint_syntax: LintSyntaxStorage,
-    pub lint_semantic: LintSemanticStorage,
-}
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use std::path::Path;
-    use std::sync::Arc;
-
-    use crate::db::{
-        Database, DbRuntime, DbWithJar, HasJar, HasJars, JarsStorage, LintDb, LintJar, QueryResult,
-        SourceDb, SourceJar, Upcast,
-    };
-    use crate::files::{FileId, Files};
-
-    use super::{SemanticDb, SemanticJar};
-
-    // This can be a partial database used in a single crate for testing.
-    // It would hold fewer data than the full database.
-    #[derive(Debug, Default)]
-    pub(crate) struct TestDb {
-        files: Files,
-        jars: JarsStorage<Self>,
-    }
-
-    impl HasJar<SourceJar> for TestDb {
-        fn jar(&self) -> QueryResult<&SourceJar> {
-            Ok(&self.jars()?.0)
-        }
-
-        fn jar_mut(&mut self) -> &mut SourceJar {
-            &mut self.jars_mut().0
-        }
-    }
-
-    impl HasJar<SemanticJar> for TestDb {
-        fn jar(&self) -> QueryResult<&SemanticJar> {
-            Ok(&self.jars()?.1)
-        }
-
-        fn jar_mut(&mut self) -> &mut SemanticJar {
-            &mut self.jars_mut().1
-        }
-    }
-
-    impl HasJar<LintJar> for TestDb {
-        fn jar(&self) -> QueryResult<&LintJar> {
-            Ok(&self.jars()?.2)
-        }
-
-        fn jar_mut(&mut self) -> &mut LintJar {
-            &mut self.jars_mut().2
-        }
-    }
-
-    impl SourceDb for TestDb {
-        fn file_id(&self, path: &Path) -> FileId {
-            self.files.intern(path)
-        }
-
-        fn file_path(&self, file_id: FileId) -> Arc<Path> {
-            self.files.path(file_id)
-        }
-    }
-
-    impl DbWithJar<SourceJar> for TestDb {}
-
-    impl Upcast<dyn SourceDb> for TestDb {
-        fn upcast(&self) -> &(dyn SourceDb + 'static) {
-            self
-        }
-    }
-
-    impl SemanticDb for TestDb {}
-
-    impl DbWithJar<SemanticJar> for TestDb {}
-
-    impl Upcast<dyn SemanticDb> for TestDb {
-        fn upcast(&self) -> &(dyn SemanticDb + 'static) {
-            self
-        }
-    }
-
-    impl LintDb for TestDb {}
-
-    impl Upcast<dyn LintDb> for TestDb {
-        fn upcast(&self) -> &(dyn LintDb + 'static) {
-            self
-        }
-    }
-
-    impl DbWithJar<LintJar> for TestDb {}
-
-    impl HasJars for TestDb {
-        type Jars = (SourceJar, SemanticJar, LintJar);
-
-        fn jars(&self) -> QueryResult<&Self::Jars> {
-            self.jars.jars()
-        }
-
-        fn jars_mut(&mut self) -> &mut Self::Jars {
-            self.jars.jars_mut()
-        }
-    }
-
-    impl Database for TestDb {
-        fn runtime(&self) -> &DbRuntime {
-            self.jars.runtime()
-        }
-
-        fn runtime_mut(&mut self) -> &mut DbRuntime {
-            self.jars.runtime_mut()
-        }
-    }
-}

crates/red_knot/src/db/jars.rs

@@ -1,37 +0,0 @@
-use crate::db::query::QueryResult;
-
-/// Gives access to a specific jar in the database.
-///
-/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
-/// which is an analogy to storing the salsa in different jars.
-///
-/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
-/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
-/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
-///
-/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
-/// We don't need this just yet because we write our queries by hand. We may want a similar trait if we decide
-/// to use a macro to generate the queries.
-pub trait HasJar<T> {
-    /// Gives a read-only reference to the jar.
-    fn jar(&self) -> QueryResult<&T>;
-
-    /// Gives a mutable reference to the jar.
-    fn jar_mut(&mut self) -> &mut T;
-}
-
-/// Gives access to the jars in a database.
-pub trait HasJars {
-    /// A type storing the jars.
-    ///
-    /// Most commonly, this is a tuple where each jar is a tuple element.
-    type Jars: Default;
-
-    /// Gives access to the underlying jars but tests if the queries have been cancelled.
-    ///
-    /// Returns `Err(QueryError::Cancelled)` if the queries have been cancelled.
-    fn jars(&self) -> QueryResult<&Self::Jars>;
-
-    /// Gives mutable access to the underlying jars.
-    fn jars_mut(&mut self) -> &mut Self::Jars;
-}

crates/red_knot/src/db/query.rs

@@ -1,20 +0,0 @@
-use std::fmt::{Display, Formatter};
-
-/// Reason why a db query operation failed.
-#[derive(Debug, Clone, Copy)]
-pub enum QueryError {
-    /// The query was cancelled because the DB was mutated or the query was cancelled by the host (e.g. on a file change or when pressing CTRL+C).
-    Cancelled,
-}
-
-impl Display for QueryError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self {
-            QueryError::Cancelled => f.write_str("query was cancelled"),
-        }
-    }
-}
-
-impl std::error::Error for QueryError {}
-
-pub type QueryResult<T> = Result<T, QueryError>;

crates/red_knot/src/db/runtime.rs

@@ -1,41 +0,0 @@
-use crate::cancellation::CancellationTokenSource;
-use crate::db::{QueryError, QueryResult};
-
-/// Holds the jar agnostic state of the database.
-#[derive(Debug, Default)]
-pub struct DbRuntime {
-    /// The cancellation token source used to signal other works that the queries should be aborted and
-    /// exit at the next possible point.
-    cancellation_token: CancellationTokenSource,
-}
-
-impl DbRuntime {
-    pub(super) fn snapshot(&self) -> Self {
-        Self {
-            cancellation_token: self.cancellation_token.clone(),
-        }
-    }
-
-    /// Cancels the pending queries of other workers. The current worker cannot have any pending
-    /// queries because we're holding a mutable reference to the runtime.
-    pub(super) fn cancel_other_workers(&mut self) {
-        self.cancellation_token.cancel();
-        // Set a new cancellation token so that we're in a non-cancelled state again when running the next
-        // query.
-        self.cancellation_token = CancellationTokenSource::default();
-    }
-
-    /// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
-    pub(super) fn cancelled(&self) -> QueryResult<()> {
-        if self.cancellation_token.is_cancelled() {
-            Err(QueryError::Cancelled)
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Returns `true` if the queries have been cancelled.
-    pub(super) fn is_cancelled(&self) -> bool {
-        self.cancellation_token.is_cancelled()
-    }
-}

crates/red_knot/src/db/storage.rs

@@ -1,117 +0,0 @@
-use std::fmt::Formatter;
-use std::sync::Arc;
-
-use crossbeam::sync::WaitGroup;
-
-use crate::db::query::QueryResult;
-use crate::db::runtime::DbRuntime;
-use crate::db::{HasJars, ParallelDatabase};
-
-/// Stores the jars of a database and the state for each worker.
-///
-/// Today, all state is shared across all workers, but it may be desired to store data per worker in the future.
-pub struct JarsStorage<T>
-where
-    T: HasJars + Sized,
-{
-    // It's important that `jars_wait_group` is declared after `jars` to ensure that `jars` is dropped first.
-    // See https://doc.rust-lang.org/reference/destructors.html
-    /// Stores the jars of the database.
-    jars: Arc<T::Jars>,
-
-    /// Used to count the references to `jars`. Allows implementing `jars_mut` without requiring to clone `jars`.
-    jars_wait_group: WaitGroup,
-
-    /// The data agnostic state.
-    runtime: DbRuntime,
-}
-
-impl<Db> JarsStorage<Db>
-where
-    Db: HasJars,
-{
-    pub(super) fn new() -> Self {
-        Self {
-            jars: Arc::new(Db::Jars::default()),
-            jars_wait_group: WaitGroup::default(),
-            runtime: DbRuntime::default(),
-        }
-    }
-
-    /// Creates a snapshot of the jars.
-    ///
-    /// Creating the snapshot is cheap because it doesn't clone the jars, it only increments a ref counter.
-    #[must_use]
-    pub fn snapshot(&self) -> JarsStorage<Db>
-    where
-        Db: ParallelDatabase,
-    {
-        Self {
-            jars: self.jars.clone(),
-            jars_wait_group: self.jars_wait_group.clone(),
-            runtime: self.runtime.snapshot(),
-        }
-    }
-
-    pub(crate) fn jars(&self) -> QueryResult<&Db::Jars> {
-        self.runtime.cancelled()?;
-        Ok(&self.jars)
-    }
-
-    /// Returns a mutable reference to the jars without cloning their content.
-    ///
-    /// The method cancels any pending queries of other works and waits for them to complete so that
-    /// this instance is the only instance holding a reference to the jars.
-    pub(crate) fn jars_mut(&mut self) -> &mut Db::Jars {
-        // We have a mutable ref here, so no more workers can be spawned between calling this function and taking the mut ref below.
-        self.cancel_other_workers();
-
-        // Now all other references to `self.jars` should have been released. We can now safely return a mutable reference
-        // to the Arc's content.
-        let jars =
-            Arc::get_mut(&mut self.jars).expect("All references to jars should have been released");
-
-        jars
-    }
-
-    pub(crate) fn runtime(&self) -> &DbRuntime {
-        &self.runtime
-    }
-
-    pub(crate) fn runtime_mut(&mut self) -> &mut DbRuntime {
-        // Note: This method may need to use a similar trick to `jars_mut` if `DbRuntime` is ever to store data that is shared between workers.
-        &mut self.runtime
-    }
-
-    #[tracing::instrument(level = "trace", skip(self))]
-    fn cancel_other_workers(&mut self) {
-        self.runtime.cancel_other_workers();
-
-        // Wait for all other works to complete.
-        let existing_wait = std::mem::take(&mut self.jars_wait_group);
-        existing_wait.wait();
-    }
-}
-
-impl<Db> Default for JarsStorage<Db>
-where
-    Db: HasJars,
-{
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<T> std::fmt::Debug for JarsStorage<T>
-where
-    T: HasJars,
-    <T as HasJars>::Jars: std::fmt::Debug,
-{
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("SharedStorage")
-            .field("jars", &self.jars)
-            .field("jars_wait_group", &self.jars_wait_group)
-            .field("runtime", &self.runtime)
-            .finish()
-    }
-}

crates/red_knot/src/files.rs

@@ -1,180 +0,0 @@
-use std::fmt::{Debug, Formatter};
-use std::hash::{Hash, Hasher};
-use std::path::Path;
-use std::sync::Arc;
-
-use hashbrown::hash_map::RawEntryMut;
-use parking_lot::RwLock;
-use rustc_hash::FxHasher;
-
-use ruff_index::{newtype_index, IndexVec};
-
-type Map<K, V> = hashbrown::HashMap<K, V, ()>;
-
-#[newtype_index]
-pub struct FileId;
-
-// TODO we'll need a higher level virtual file system abstraction that allows testing if a file exists
-//  or retrieving its content (ideally lazily and in a way that the memory can be retained later)
-//  I suspect that we'll end up with a FileSystem trait and our own Path abstraction.
-#[derive(Default)]
-pub struct Files {
-    inner: Arc<RwLock<FilesInner>>,
-}
-
-impl Files {
-    #[tracing::instrument(level = "debug", skip(self))]
-    pub fn intern(&self, path: &Path) -> FileId {
-        self.inner.write().intern(path)
-    }
-
-    pub fn try_get(&self, path: &Path) -> Option<FileId> {
-        self.inner.read().try_get(path)
-    }
-
-    #[tracing::instrument(level = "debug", skip(self))]
-    pub fn path(&self, id: FileId) -> Arc<Path> {
-        self.inner.read().path(id)
-    }
-
-    /// Snapshots files for a new database snapshot.
-    ///
-    /// This method should not be used outside a database snapshot.
-    #[must_use]
-    pub fn snapshot(&self) -> Files {
-        Files {
-            inner: self.inner.clone(),
-        }
-    }
-}
-
-impl Debug for Files {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let files = self.inner.read();
-        let mut debug = f.debug_map();
-        for item in files.iter() {
-            debug.entry(&item.0, &item.1);
-        }
-
-        debug.finish()
-    }
-}
-
-impl PartialEq for Files {
-    fn eq(&self, other: &Self) -> bool {
-        self.inner.read().eq(&other.inner.read())
-    }
-}
-
-impl Eq for Files {}
-
-#[derive(Default)]
-struct FilesInner {
-    by_path: Map<FileId, ()>,
-    // TODO should we use a map here to reclaim the space for removed files?
-    // TODO I think we should use our own path abstraction here to avoid having to normalize paths
-    //  and dealing with non-utf paths everywhere.
-    by_id: IndexVec<FileId, Arc<Path>>,
-}
-
-impl FilesInner {
-    /// Inserts the path and returns a new id for it or returns the id if it is an existing path.
-    // TODO should this accept Path or PathBuf?
-    pub(crate) fn intern(&mut self, path: &Path) -> FileId {
-        let hash = FilesInner::hash_path(path);
-
-        let entry = self
-            .by_path
-            .raw_entry_mut()
-            .from_hash(hash, |existing_file| &*self.by_id[*existing_file] == path);
-
-        match entry {
-            RawEntryMut::Occupied(entry) => *entry.key(),
-            RawEntryMut::Vacant(entry) => {
-                let id = self.by_id.push(Arc::from(path));
-                entry.insert_with_hasher(hash, id, (), |file| {
-                    FilesInner::hash_path(&self.by_id[*file])
-                });
-                id
-            }
-        }
-    }
-
-    fn hash_path(path: &Path) -> u64 {
-        let mut hasher = FxHasher::default();
-        path.hash(&mut hasher);
-        hasher.finish()
-    }
-
-    pub(crate) fn try_get(&self, path: &Path) -> Option<FileId> {
-        let mut hasher = FxHasher::default();
-        path.hash(&mut hasher);
-        let hash = hasher.finish();
-
-        Some(
-            *self
-                .by_path
-                .raw_entry()
-                .from_hash(hash, |existing_file| &*self.by_id[*existing_file] == path)?
-                .0,
-        )
-    }
-
-    /// Returns the path for the file with the given id.
-    pub(crate) fn path(&self, id: FileId) -> Arc<Path> {
-        self.by_id[id].clone()
-    }
-
-    pub(crate) fn iter(&self) -> impl Iterator<Item = (FileId, Arc<Path>)> + '_ {
-        self.by_path.keys().map(|id| (*id, self.by_id[*id].clone()))
-    }
-}
-
-impl PartialEq for FilesInner {
-    fn eq(&self, other: &Self) -> bool {
-        self.by_id == other.by_id
-    }
-}
-
-impl Eq for FilesInner {}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::path::PathBuf;
-
-    #[test]
-    fn insert_path_twice_same_id() {
-        let files = Files::default();
-        let path = PathBuf::from("foo/bar");
-        let id1 = files.intern(&path);
-        let id2 = files.intern(&path);
-        assert_eq!(id1, id2);
-    }
-
-    #[test]
-    fn insert_different_paths_different_ids() {
-        let files = Files::default();
-        let path1 = PathBuf::from("foo/bar");
-        let path2 = PathBuf::from("foo/bar/baz");
-        let id1 = files.intern(&path1);
-        let id2 = files.intern(&path2);
-        assert_ne!(id1, id2);
-    }
-
-    #[test]
-    fn four_files() {
-        let files = Files::default();
-        let foo_path = PathBuf::from("foo");
-        let foo_id = files.intern(&foo_path);
-        let bar_path = PathBuf::from("bar");
-        files.intern(&bar_path);
-        let baz_path = PathBuf::from("baz");
-        files.intern(&baz_path);
-        let qux_path = PathBuf::from("qux");
-        files.intern(&qux_path);
-
-        let foo_id_2 = files.try_get(&foo_path).expect("foo_path to be found");
-        assert_eq!(foo_id_2, foo_id);
-    }
-}

crates/red_knot/src/hir.rs

@@ -1,67 +0,0 @@
-//! Key observations
-//!
-//! The HIR (High-Level Intermediate Representation) avoids allocations to large extends by:
-//! * Using an arena per node type
-//! * using ids and id ranges to reference items.
-//!
-//! Using separate arena per node type has the advantage that the IDs are relatively stable, because
-//! they only change when a node of the same kind has been added or removed. (What's unclear is if that matters or if
-//! it still triggers a re-compute because the AST-id in the node has changed).
-//!
-//! The HIR does not store all details. It mainly stores the *public* interface. There's a reference
-//! back to the AST node to get more details.
-//!
-//!
-
-use crate::ast_ids::{HasAstId, TypedAstId};
-use crate::files::FileId;
-use std::fmt::Formatter;
-use std::hash::{Hash, Hasher};
-
-pub struct HirAstId<N: HasAstId> {
-    file_id: FileId,
-    node_id: TypedAstId<N>,
-}
-
-impl<N: HasAstId> Copy for HirAstId<N> {}
-impl<N: HasAstId> Clone for HirAstId<N> {
-    fn clone(&self) -> Self {
-        *self
-    }
-}
-
-impl<N: HasAstId> PartialEq for HirAstId<N> {
-    fn eq(&self, other: &Self) -> bool {
-        self.file_id == other.file_id && self.node_id == other.node_id
-    }
-}
-
-impl<N: HasAstId> Eq for HirAstId<N> {}
-
-impl<N: HasAstId> std::fmt::Debug for HirAstId<N> {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("HirAstId")
-            .field("file_id", &self.file_id)
-            .field("node_id", &self.node_id)
-            .finish()
-    }
-}
-
-impl<N: HasAstId> Hash for HirAstId<N> {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.file_id.hash(state);
-        self.node_id.hash(state);
-    }
-}
-
-impl<N: HasAstId> HirAstId<N> {
-    pub fn upcast<M: HasAstId>(self) -> HirAstId<M>
-    where
-        N: Into<M>,
-    {
-        HirAstId {
-            file_id: self.file_id,
-            node_id: self.node_id.upcast(),
-        }
-    }
-}

crates/red_knot/src/hir/definition.rs

@@ -1,556 +0,0 @@
-use std::ops::{Index, Range};
-
-use ruff_index::{newtype_index, IndexVec};
-use ruff_python_ast::visitor::preorder;
-use ruff_python_ast::visitor::preorder::PreorderVisitor;
-use ruff_python_ast::{
-    Decorator, ExceptHandler, ExceptHandlerExceptHandler, Expr, MatchCase, ModModule, Stmt,
-    StmtAnnAssign, StmtAssign, StmtClassDef, StmtFunctionDef, StmtGlobal, StmtImport,
-    StmtImportFrom, StmtNonlocal, StmtTypeAlias, TypeParam, TypeParamParamSpec, TypeParamTypeVar,
-    TypeParamTypeVarTuple, WithItem,
-};
-
-use crate::ast_ids::{AstIds, HasAstId};
-use crate::files::FileId;
-use crate::hir::HirAstId;
-use crate::Name;
-
-#[newtype_index]
-pub struct FunctionId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Function {
-    ast_id: HirAstId<StmtFunctionDef>,
-    name: Name,
-    parameters: Range<ParameterId>,
-    type_parameters: Range<TypeParameterId>, // TODO: type_parameters, return expression, decorators
-}
-
-#[newtype_index]
-pub struct ParameterId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Parameter {
-    kind: ParameterKind,
-    name: Name,
-    default: Option<()>, // TODO use expression HIR
-    ast_id: HirAstId<ruff_python_ast::Parameter>,
-}
-
-// TODO or should `Parameter` be an enum?
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
-pub enum ParameterKind {
-    PositionalOnly,
-    Arguments,
-    Vararg,
-    KeywordOnly,
-    Kwarg,
-}
-
-#[newtype_index]
-pub struct ClassId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Class {
-    name: Name,
-    ast_id: HirAstId<StmtClassDef>,
-    // TODO type parameters, inheritance, decorators, members
-}
-
-#[newtype_index]
-pub struct AssignmentId;
-
-// This can have more than one name...
-// but that means we can't implement `name()` on `ModuleItem`.
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Assignment {
-    // TODO: Handle multiple names  / targets
-    name: Name,
-    ast_id: HirAstId<StmtAssign>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct AnnotatedAssignment {
-    name: Name,
-    ast_id: HirAstId<StmtAnnAssign>,
-}
-
-#[newtype_index]
-pub struct AnnotatedAssignmentId;
-
-#[newtype_index]
-pub struct TypeAliasId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct TypeAlias {
-    name: Name,
-    ast_id: HirAstId<StmtTypeAlias>,
-    parameters: Range<TypeParameterId>,
-}
-
-#[newtype_index]
-pub struct TypeParameterId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum TypeParameter {
-    TypeVar(TypeParameterTypeVar),
-    ParamSpec(TypeParameterParamSpec),
-    TypeVarTuple(TypeParameterTypeVarTuple),
-}
-
-impl TypeParameter {
-    pub fn ast_id(&self) -> HirAstId<TypeParam> {
-        match self {
-            TypeParameter::TypeVar(type_var) => type_var.ast_id.upcast(),
-            TypeParameter::ParamSpec(param_spec) => param_spec.ast_id.upcast(),
-            TypeParameter::TypeVarTuple(type_var_tuple) => type_var_tuple.ast_id.upcast(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct TypeParameterTypeVar {
-    name: Name,
-    ast_id: HirAstId<TypeParamTypeVar>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct TypeParameterParamSpec {
-    name: Name,
-    ast_id: HirAstId<TypeParamParamSpec>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct TypeParameterTypeVarTuple {
-    name: Name,
-    ast_id: HirAstId<TypeParamTypeVarTuple>,
-}
-
-#[newtype_index]
-pub struct GlobalId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Global {
-    // TODO track names
-    ast_id: HirAstId<StmtGlobal>,
-}
-
-#[newtype_index]
-pub struct NonLocalId;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct NonLocal {
-    // TODO track names
-    ast_id: HirAstId<StmtNonlocal>,
-}
-
-pub enum DefinitionId {
-    Function(FunctionId),
-    Parameter(ParameterId),
-    Class(ClassId),
-    Assignment(AssignmentId),
-    AnnotatedAssignment(AnnotatedAssignmentId),
-    Global(GlobalId),
-    NonLocal(NonLocalId),
-    TypeParameter(TypeParameterId),
-    TypeAlias(TypeAlias),
-}
-
-pub enum DefinitionItem {
-    Function(Function),
-    Parameter(Parameter),
-    Class(Class),
-    Assignment(Assignment),
-    AnnotatedAssignment(AnnotatedAssignment),
-    Global(Global),
-    NonLocal(NonLocal),
-    TypeParameter(TypeParameter),
-    TypeAlias(TypeAlias),
-}
-
-// The closest is rust-analyzers item-tree. It only represents "Items" which make the public interface of a module
-// (it excludes any other statement or expressions). rust-analyzer uses it as the main input to the name resolution
-// algorithm
-// > It is the input to the name resolution algorithm, as well as to the queries defined in `adt.rs`,
-// > `data.rs`, and most things in `attr.rs`.
-//
-// > One important purpose of this layer is to provide an "invalidation barrier" for incremental
-// > computations: when typing inside an item body, the `ItemTree` of the modified file is typically
-// > unaffected, so we don't have to recompute name resolution results or item data (see `data.rs`).
-//
-// I haven't fully figured this out but I think that this composes the "public" interface of a module?
-// But maybe that's too optimistic.
-//
-//
-#[derive(Debug, Clone, Default, Eq, PartialEq)]
-pub struct Definitions {
-    functions: IndexVec<FunctionId, Function>,
-    parameters: IndexVec<ParameterId, Parameter>,
-    classes: IndexVec<ClassId, Class>,
-    assignments: IndexVec<AssignmentId, Assignment>,
-    annotated_assignments: IndexVec<AnnotatedAssignmentId, AnnotatedAssignment>,
-    type_aliases: IndexVec<TypeAliasId, TypeAlias>,
-    type_parameters: IndexVec<TypeParameterId, TypeParameter>,
-    globals: IndexVec<GlobalId, Global>,
-    non_locals: IndexVec<NonLocalId, NonLocal>,
-}
-
-impl Definitions {
-    pub fn from_module(module: &ModModule, ast_ids: &AstIds, file_id: FileId) -> Self {
-        let mut visitor = DefinitionsVisitor {
-            definitions: Definitions::default(),
-            ast_ids,
-            file_id,
-        };
-
-        visitor.visit_body(&module.body);
-
-        visitor.definitions
-    }
-}
-
-impl Index<FunctionId> for Definitions {
-    type Output = Function;
-
-    fn index(&self, index: FunctionId) -> &Self::Output {
-        &self.functions[index]
-    }
-}
-
-impl Index<ParameterId> for Definitions {
-    type Output = Parameter;
-
-    fn index(&self, index: ParameterId) -> &Self::Output {
-        &self.parameters[index]
-    }
-}
-
-impl Index<ClassId> for Definitions {
-    type Output = Class;
-
-    fn index(&self, index: ClassId) -> &Self::Output {
-        &self.classes[index]
-    }
-}
-
-impl Index<AssignmentId> for Definitions {
-    type Output = Assignment;
-
-    fn index(&self, index: AssignmentId) -> &Self::Output {
-        &self.assignments[index]
-    }
-}
-
-impl Index<AnnotatedAssignmentId> for Definitions {
-    type Output = AnnotatedAssignment;
-
-    fn index(&self, index: AnnotatedAssignmentId) -> &Self::Output {
-        &self.annotated_assignments[index]
-    }
-}
-
-impl Index<TypeAliasId> for Definitions {
-    type Output = TypeAlias;
-
-    fn index(&self, index: TypeAliasId) -> &Self::Output {
-        &self.type_aliases[index]
-    }
-}
-
-impl Index<GlobalId> for Definitions {
-    type Output = Global;
-
-    fn index(&self, index: GlobalId) -> &Self::Output {
-        &self.globals[index]
-    }
-}
-
-impl Index<NonLocalId> for Definitions {
-    type Output = NonLocal;
-
-    fn index(&self, index: NonLocalId) -> &Self::Output {
-        &self.non_locals[index]
-    }
-}
-
-impl Index<TypeParameterId> for Definitions {
-    type Output = TypeParameter;
-
-    fn index(&self, index: TypeParameterId) -> &Self::Output {
-        &self.type_parameters[index]
-    }
-}
-
-struct DefinitionsVisitor<'a> {
-    definitions: Definitions,
-    ast_ids: &'a AstIds,
-    file_id: FileId,
-}
-
-impl DefinitionsVisitor<'_> {
-    fn ast_id<N: HasAstId>(&self, node: &N) -> HirAstId<N> {
-        HirAstId {
-            file_id: self.file_id,
-            node_id: self.ast_ids.ast_id(node),
-        }
-    }
-
-    fn lower_function_def(&mut self, function: &StmtFunctionDef) -> FunctionId {
-        let name = Name::new(&function.name);
-
-        let first_type_parameter_id = self.definitions.type_parameters.next_index();
-        let mut last_type_parameter_id = first_type_parameter_id;
-
-        if let Some(type_params) = &function.type_params {
-            for parameter in &type_params.type_params {
-                let id = self.lower_type_parameter(parameter);
-                last_type_parameter_id = id;
-            }
-        }
-
-        let parameters = self.lower_parameters(&function.parameters);
-
-        self.definitions.functions.push(Function {
-            name,
-            ast_id: self.ast_id(function),
-            parameters,
-            type_parameters: first_type_parameter_id..last_type_parameter_id,
-        })
-    }
-
-    fn lower_parameters(&mut self, parameters: &ruff_python_ast::Parameters) -> Range<ParameterId> {
-        let first_parameter_id = self.definitions.parameters.next_index();
-        let mut last_parameter_id = first_parameter_id;
-
-        for parameter in &parameters.posonlyargs {
-            last_parameter_id = self.definitions.parameters.push(Parameter {
-                kind: ParameterKind::PositionalOnly,
-                name: Name::new(&parameter.parameter.name),
-                default: None,
-                ast_id: self.ast_id(&parameter.parameter),
-            });
-        }
-
-        if let Some(vararg) = &parameters.vararg {
-            last_parameter_id = self.definitions.parameters.push(Parameter {
-                kind: ParameterKind::Vararg,
-                name: Name::new(&vararg.name),
-                default: None,
-                ast_id: self.ast_id(vararg),
-            });
-        }
-
-        for parameter in &parameters.kwonlyargs {
-            last_parameter_id = self.definitions.parameters.push(Parameter {
-                kind: ParameterKind::KeywordOnly,
-                name: Name::new(&parameter.parameter.name),
-                default: None,
-                ast_id: self.ast_id(&parameter.parameter),
-            });
-        }
-
-        if let Some(kwarg) = &parameters.kwarg {
-            last_parameter_id = self.definitions.parameters.push(Parameter {
-                kind: ParameterKind::KeywordOnly,
-                name: Name::new(&kwarg.name),
-                default: None,
-                ast_id: self.ast_id(kwarg),
-            });
-        }
-
-        first_parameter_id..last_parameter_id
-    }
-
-    fn lower_class_def(&mut self, class: &StmtClassDef) -> ClassId {
-        let name = Name::new(&class.name);
-
-        self.definitions.classes.push(Class {
-            name,
-            ast_id: self.ast_id(class),
-        })
-    }
-
-    fn lower_assignment(&mut self, assignment: &StmtAssign) {
-        // FIXME handle multiple names
-        if let Some(Expr::Name(name)) = assignment.targets.first() {
-            self.definitions.assignments.push(Assignment {
-                name: Name::new(&name.id),
-                ast_id: self.ast_id(assignment),
-            });
-        }
-    }
-
-    fn lower_annotated_assignment(&mut self, annotated_assignment: &StmtAnnAssign) {
-        if let Expr::Name(name) = &*annotated_assignment.target {
-            self.definitions
-                .annotated_assignments
-                .push(AnnotatedAssignment {
-                    name: Name::new(&name.id),
-                    ast_id: self.ast_id(annotated_assignment),
-                });
-        }
-    }
-
-    fn lower_type_alias(&mut self, type_alias: &StmtTypeAlias) {
-        if let Expr::Name(name) = &*type_alias.name {
-            let name = Name::new(&name.id);
-
-            let lower_parameters_id = self.definitions.type_parameters.next_index();
-            let mut last_parameter_id = lower_parameters_id;
-
-            if let Some(type_params) = &type_alias.type_params {
-                for type_parameter in &type_params.type_params {
-                    let id = self.lower_type_parameter(type_parameter);
-                    last_parameter_id = id;
-                }
-            }
-
-            self.definitions.type_aliases.push(TypeAlias {
-                name,
-                ast_id: self.ast_id(type_alias),
-                parameters: lower_parameters_id..last_parameter_id,
-            });
-        }
-    }
-
-    fn lower_type_parameter(&mut self, type_parameter: &TypeParam) -> TypeParameterId {
-        match type_parameter {
-            TypeParam::TypeVar(type_var) => {
-                self.definitions
-                    .type_parameters
-                    .push(TypeParameter::TypeVar(TypeParameterTypeVar {
-                        name: Name::new(&type_var.name),
-                        ast_id: self.ast_id(type_var),
-                    }))
-            }
-            TypeParam::ParamSpec(param_spec) => {
-                self.definitions
-                    .type_parameters
-                    .push(TypeParameter::ParamSpec(TypeParameterParamSpec {
-                        name: Name::new(&param_spec.name),
-                        ast_id: self.ast_id(param_spec),
-                    }))
-            }
-            TypeParam::TypeVarTuple(type_var_tuple) => {
-                self.definitions
-                    .type_parameters
-                    .push(TypeParameter::TypeVarTuple(TypeParameterTypeVarTuple {
-                        name: Name::new(&type_var_tuple.name),
-                        ast_id: self.ast_id(type_var_tuple),
-                    }))
-            }
-        }
-    }
-
-    fn lower_import(&mut self, _import: &StmtImport) {
-        // TODO
-    }
-
-    fn lower_import_from(&mut self, _import_from: &StmtImportFrom) {
-        // TODO
-    }
-
-    fn lower_global(&mut self, global: &StmtGlobal) -> GlobalId {
-        self.definitions.globals.push(Global {
-            ast_id: self.ast_id(global),
-        })
-    }
-
-    fn lower_non_local(&mut self, non_local: &StmtNonlocal) -> NonLocalId {
-        self.definitions.non_locals.push(NonLocal {
-            ast_id: self.ast_id(non_local),
-        })
-    }
-
-    fn lower_except_handler(&mut self, _except_handler: &ExceptHandlerExceptHandler) {
-        // TODO
-    }
-
-    fn lower_with_item(&mut self, _with_item: &WithItem) {
-        // TODO
-    }
-
-    fn lower_match_case(&mut self, _match_case: &MatchCase) {
-        // TODO
-    }
-}
-
-impl PreorderVisitor<'_> for DefinitionsVisitor<'_> {
-    fn visit_stmt(&mut self, stmt: &Stmt) {
-        match stmt {
-            // Definition statements
-            Stmt::FunctionDef(definition) => {
-                self.lower_function_def(definition);
-                self.visit_body(&definition.body);
-            }
-            Stmt::ClassDef(definition) => {
-                self.lower_class_def(definition);
-                self.visit_body(&definition.body);
-            }
-            Stmt::Assign(assignment) => {
-                self.lower_assignment(assignment);
-            }
-            Stmt::AnnAssign(annotated_assignment) => {
-                self.lower_annotated_assignment(annotated_assignment);
-            }
-            Stmt::TypeAlias(type_alias) => {
-                self.lower_type_alias(type_alias);
-            }
-
-            Stmt::Import(import) => self.lower_import(import),
-            Stmt::ImportFrom(import_from) => self.lower_import_from(import_from),
-            Stmt::Global(global) => {
-                self.lower_global(global);
-            }
-            Stmt::Nonlocal(non_local) => {
-                self.lower_non_local(non_local);
-            }
-
-            // Visit the compound statement bodies because they can contain other definitions.
-            Stmt::For(_)
-            | Stmt::While(_)
-            | Stmt::If(_)
-            | Stmt::With(_)
-            | Stmt::Match(_)
-            | Stmt::Try(_) => {
-                preorder::walk_stmt(self, stmt);
-            }
-
-            // Skip over simple statements because they can't contain any other definitions.
-            Stmt::Return(_)
-            | Stmt::Delete(_)
-            | Stmt::AugAssign(_)
-            | Stmt::Raise(_)
-            | Stmt::Assert(_)
-            | Stmt::Expr(_)
-            | Stmt::Pass(_)
-            | Stmt::Break(_)
-            | Stmt::Continue(_)
-            | Stmt::IpyEscapeCommand(_) => {
-                // No op
-            }
-        }
-    }
-
-    fn visit_expr(&mut self, _: &'_ Expr) {}
-
-    fn visit_decorator(&mut self, _decorator: &'_ Decorator) {}
-
-    fn visit_except_handler(&mut self, except_handler: &'_ ExceptHandler) {
-        match except_handler {
-            ExceptHandler::ExceptHandler(except_handler) => {
-                self.lower_except_handler(except_handler);
-            }
-        }
-    }
-
-    fn visit_with_item(&mut self, with_item: &'_ WithItem) {
-        self.lower_with_item(with_item);
-    }
-
-    fn visit_match_case(&mut self, match_case: &'_ MatchCase) {
-        self.lower_match_case(match_case);
-        self.visit_body(&match_case.body);
-    }
-}

crates/red_knot/src/lib.rs

@@ -1,68 +1,52 @@
-use std::hash::BuildHasherDefault;
+use rustc_hash::FxHashSet;
-use std::path::{Path, PathBuf};
 
-use rustc_hash::{FxHashSet, FxHasher};
+use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};
+use ruff_db::vfs::VfsFile;
 
-use crate::files::FileId;
+use crate::db::Jar;
 
-pub mod ast_ids;
-pub mod cache;
-pub mod cancellation;
 pub mod db;
-pub mod files;
-pub mod hir;
 pub mod lint;
-pub mod module;
-mod parse;
 pub mod program;
-mod semantic;
-pub mod source;
 pub mod watch;
 
-pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHasher>>;
-#[allow(unused)]
-pub(crate) type FxDashSet<V> = dashmap::DashSet<V, BuildHasherDefault<FxHasher>>;
-pub(crate) type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;
-
 #[derive(Debug, Clone)]
 pub struct Workspace {
-    /// TODO this should be a resolved path. We should probably use a newtype wrapper that guarantees that
+    root: FileSystemPathBuf,
-    /// PATH is a UTF-8 path and is normalized.
-    root: PathBuf,
     /// The files that are open in the workspace.
     ///
     /// * Editor: The files that are actively being edited in the editor (the user has a tab open with the file).
     /// * CLI: The resolved files passed as arguments to the CLI.
-    open_files: FxHashSet<FileId>,
+    open_files: FxHashSet<VfsFile>,
 }
 
 impl Workspace {
-    pub fn new(root: PathBuf) -> Self {
+    pub fn new(root: FileSystemPathBuf) -> Self {
         Self {
             root,
             open_files: FxHashSet::default(),
         }
     }
 
-    pub fn root(&self) -> &Path {
+    pub fn root(&self) -> &FileSystemPath {
         self.root.as_path()
     }
 
     // TODO having the content in workspace feels wrong.
-    pub fn open_file(&mut self, file_id: FileId) {
+    pub fn open_file(&mut self, file_id: VfsFile) {
         self.open_files.insert(file_id);
     }
 
-    pub fn close_file(&mut self, file_id: FileId) {
+    pub fn close_file(&mut self, file_id: VfsFile) {
         self.open_files.remove(&file_id);
     }
 
     // TODO introduce an `OpenFile` type instead of using an anonymous tuple.
-    pub fn open_files(&self) -> impl Iterator<Item = FileId> + '_ {
+    pub fn open_files(&self) -> impl Iterator<Item = VfsFile> + '_ {
         self.open_files.iter().copied()
     }
 
-    pub fn is_file_open(&self, file_id: FileId) -> bool {
+    pub fn is_file_open(&self, file_id: VfsFile) -> bool {
         self.open_files.contains(&file_id)
     }
 }

crates/red_knot/src/lint.rs

@@ -1,61 +1,59 @@
-use red_knot_module_resolver::ModuleName;
 use std::cell::RefCell;
-use std::ops::{Deref, DerefMut};
+use std::ops::Deref;
-use std::sync::Arc;
 use std::time::Duration;
 
-use ruff_python_ast::visitor::Visitor;
+use tracing::trace_span;
-use ruff_python_ast::{ModModule, StringLiteral};
-use ruff_python_parser::Parsed;
 
-use crate::cache::KeyValueCache;
+use red_knot_module_resolver::ModuleName;
-use crate::db::{LintDb, LintJar, QueryResult};
+use red_knot_python_semantic::types::Type;
-use crate::files::FileId;
+use red_knot_python_semantic::{HasTy, SemanticModel};
-use crate::module::resolve_module;
+use ruff_db::parsed::{parsed_module, ParsedModule};
-use crate::parse::parse;
+use ruff_db::source::{source_text, SourceText};
-use crate::semantic::{infer_definition_type, infer_symbol_public_type, Type};
+use ruff_db::vfs::VfsFile;
-use crate::semantic::{
+use ruff_python_ast as ast;
-    resolve_global_symbol, semantic_index, Definition, GlobalSymbolId, SemanticIndex, SymbolId,
+use ruff_python_ast::visitor::{walk_stmt, Visitor};
-};
-use crate::source::{source_text, Source};
 
-#[tracing::instrument(level = "debug", skip(db))]
+use crate::db::Db;
-pub(crate) fn lint_syntax(db: &dyn LintDb, file_id: FileId) -> QueryResult<Diagnostics> {
-    let lint_jar: &LintJar = db.jar()?;
-    let storage = &lint_jar.lint_syntax;
 
+/// Workaround query to test for if the computation should be cancelled.
+/// Ideally, push for Salsa to expose an API for testing if cancellation was requested.
+#[salsa::tracked]
+#[allow(unused_variables)]
+pub(crate) fn unwind_if_cancelled(db: &dyn Db) {}
+
+#[salsa::tracked(return_ref)]
+pub(crate) fn lint_syntax(db: &dyn Db, file_id: VfsFile) -> Diagnostics {
     #[allow(clippy::print_stdout)]
     if std::env::var("RED_KNOT_SLOW_LINT").is_ok() {
         for i in 0..10 {
-            db.cancelled()?;
+            unwind_if_cancelled(db);
+
             println!("RED_KNOT_SLOW_LINT is set, sleeping for {i}/10 seconds");
             std::thread::sleep(Duration::from_secs(1));
         }
     }
 
-    storage.get(&file_id, |file_id| {
     let mut diagnostics = Vec::new();
 
-        let source = source_text(db.upcast(), *file_id)?;
+    let source = source_text(db.upcast(), file_id);
-        lint_lines(source.text(), &mut diagnostics);
+    lint_lines(&source, &mut diagnostics);
 
-        let parsed = parse(db.upcast(), *file_id)?;
+    let parsed = parsed_module(db.upcast(), file_id);
 
     if parsed.errors().is_empty() {
         let ast = parsed.syntax();
 
         let mut visitor = SyntaxLintVisitor {
             diagnostics,
-                source: source.text(),
+            source: &source,
         };
         visitor.visit_body(&ast.body);
         diagnostics = visitor.diagnostics;
     } else {
-            diagnostics.extend(parsed.errors().iter().map(std::string::ToString::to_string));
+        diagnostics.extend(parsed.errors().iter().map(ToString::to_string));
     }
 
-        Ok(Diagnostics::from(diagnostics))
+    Diagnostics::from(diagnostics)
-    })
 }
 
 fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
@@ -75,179 +73,127 @@ fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
     }
 }
 
-#[tracing::instrument(level = "debug", skip(db))]
+#[salsa::tracked(return_ref)]
-pub(crate) fn lint_semantic(db: &dyn LintDb, file_id: FileId) -> QueryResult<Diagnostics> {
+pub(crate) fn lint_semantic(db: &dyn Db, file_id: VfsFile) -> Diagnostics {
-    let lint_jar: &LintJar = db.jar()?;
+    let _span = trace_span!("lint_semantic", ?file_id).entered();
-    let storage = &lint_jar.lint_semantic;
 
-    storage.get(&file_id, |file_id| {
+    let source = source_text(db.upcast(), file_id);
-        let source = source_text(db.upcast(), *file_id)?;
+    let parsed = parsed_module(db.upcast(), file_id);
-        let parsed = parse(db.upcast(), *file_id)?;
+    let semantic = SemanticModel::new(db.upcast(), file_id);
-        let semantic_index = semantic_index(db.upcast(), *file_id)?;
+
+    if !parsed.is_valid() {
+        return Diagnostics::Empty;
+    }
 
     let context = SemanticLintContext {
-            file_id: *file_id,
         source,
-            parsed: &parsed,
+        parsed,
-            semantic_index,
+        semantic,
-            db,
         diagnostics: RefCell::new(Vec::new()),
     };
 
-        lint_unresolved_imports(&context)?;
+    SemanticVisitor { context: &context }.visit_body(parsed.suite());
-        lint_bad_overrides(&context)?;
 
-        Ok(Diagnostics::from(context.diagnostics.take()))
+    Diagnostics::from(context.diagnostics.take())
-    })
 }
 
-fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
+fn lint_unresolved_imports(context: &SemanticLintContext, import: AnyImportRef) {
-    // TODO: Consider iterating over the dependencies (imports) only instead of all definitions.
+    match import {
-    for (symbol, definition) in context.semantic_index().symbol_table().all_definitions() {
+        AnyImportRef::Import(import) => {
-        match definition {
+            for alias in &import.names {
-            Definition::Import(import) => {
+                let ty = alias.ty(&context.semantic);
-                let ty = context.infer_symbol_public_type(symbol)?;
 
                 if ty.is_unknown() {
-                    context.push_diagnostic(format!("Unresolved module {}", import.module));
+                    context.push_diagnostic(format!("Unresolved import '{}'", &alias.name));
                 }
             }
-            Definition::ImportFrom(import) => {
+        }
-                let ty = context.infer_symbol_public_type(symbol)?;
+        AnyImportRef::ImportFrom(import) => {
+            for alias in &import.names {
+                let ty = alias.ty(&context.semantic);
 
                 if ty.is_unknown() {
-                    let module_name = import.module().map(Deref::deref).unwrap_or_default();
+                    context.push_diagnostic(format!("Unresolved import '{}'", &alias.name));
-                    let message = if import.level() > 0 {
+                }
-                        format!(
-                            "Unresolved relative import '{}' from {}{}",
-                            import.name(),
-                            ".".repeat(import.level() as usize),
-                            module_name
-                        )
-                    } else {
-                        format!(
-                            "Unresolved import '{}' from '{}'",
-                            import.name(),
-                            module_name
-                        )
-                    };
-
-                    context.push_diagnostic(message);
             }
         }
-            _ => {}
     }
 }
 
-    Ok(())
+fn lint_bad_override(context: &SemanticLintContext, class: &ast::StmtClassDef) {
-}
+    let semantic = &context.semantic;
+    let typing_context = semantic.typing_context();
 
-fn lint_bad_overrides(context: &SemanticLintContext) -> QueryResult<()> {
     // TODO we should have a special marker on the real typing module (from typeshed) so if you
     //   have your own "typing" module in your project, we don't consider it THE typing module (and
     //   same for other stdlib modules that our lint rules care about)
-    let Some(typing_override) =
+    let Some(typing) = semantic.resolve_module(ModuleName::new("typing").unwrap()) else {
-        context.resolve_global_symbol(&ModuleName::new_static("typing").unwrap(), "override")?
+        return;
-    else {
-        // TODO once we bundle typeshed, this should be unreachable!()
-        return Ok(());
     };
 
-    // TODO we should maybe index definitions by type instead of iterating all, or else iterate all
+    let Some(typing_override) = semantic.public_symbol(&typing, "override") else {
-    // just once, match, and branch to all lint rules that care about a type of definition
+        return;
-    for (symbol, definition) in context.semantic_index().symbol_table().all_definitions() {
-        if !matches!(definition, Definition::FunctionDef(_)) {
-            continue;
-        }
-        let ty = infer_definition_type(
-            context.db.upcast(),
-            GlobalSymbolId {
-                file_id: context.file_id,
-                symbol_id: symbol,
-            },
-            definition.clone(),
-        )?;
-        let Type::Function(func) = ty else {
-            unreachable!("type of a FunctionDef should always be a Function");
     };
-        let Some(class) = func.get_containing_class(context.db.upcast())? else {
+
-            // not a method of a class
+    let override_ty = semantic.public_symbol_ty(typing_override);
-            continue;
+
+    let Type::Class(class_ty) = class.ty(semantic) else {
+        return;
     };
-        if func.has_decorator(context.db.upcast(), typing_override)? {
+
-            let method_name = func.name(context.db.upcast())?;
+    for function in class
-            if class
+        .body
-                .get_super_class_member(context.db.upcast(), &method_name)?
+        .iter()
+        .filter_map(|stmt| stmt.as_function_def_stmt())
+    {
+        let Type::Function(ty) = function.ty(semantic) else {
+            return;
+        };
+
+        if ty.has_decorator(&typing_context, override_ty) {
+            let method_name = ty.name(&typing_context);
+            if class_ty
+                .inherited_class_member(&typing_context, method_name)
                 .is_none()
             {
                 // TODO should have a qualname() method to support nested classes
                 context.push_diagnostic(
                     format!(
                         "Method {}.{} is decorated with `typing.override` but does not override any base class method",
-                        class.name(context.db.upcast())?,
+                        class_ty.name(&typing_context),
                         method_name,
                     ));
             }
         }
     }
-    Ok(())
 }
 
-pub struct SemanticLintContext<'a> {
+pub(crate) struct SemanticLintContext<'a> {
-    file_id: FileId,
+    source: SourceText,
-    source: Source,
+    parsed: &'a ParsedModule,
-    parsed: &'a Parsed<ModModule>,
+    semantic: SemanticModel<'a>,
-    semantic_index: Arc<SemanticIndex>,
-    db: &'a dyn LintDb,
     diagnostics: RefCell<Vec<String>>,
 }
 
-impl<'a> SemanticLintContext<'a> {
+impl<'db> SemanticLintContext<'db> {
-    pub fn source_text(&self) -> &str {
+    #[allow(unused)]
-        self.source.text()
+    pub(crate) fn source_text(&self) -> &str {
+        self.source.as_str()
     }
 
-    pub fn file_id(&self) -> FileId {
+    #[allow(unused)]
-        self.file_id
+    pub(crate) fn ast(&self) -> &'db ast::ModModule {
-    }
-
-    pub fn ast(&self) -> &'a ModModule {
         self.parsed.syntax()
     }
 
-    pub fn semantic_index(&self) -> &SemanticIndex {
+    pub(crate) fn push_diagnostic(&self, diagnostic: String) {
-        &self.semantic_index
-    }
-
-    pub fn infer_symbol_public_type(&self, symbol_id: SymbolId) -> QueryResult<Type> {
-        infer_symbol_public_type(
-            self.db.upcast(),
-            GlobalSymbolId {
-                file_id: self.file_id,
-                symbol_id,
-            },
-        )
-    }
-
-    pub fn push_diagnostic(&self, diagnostic: String) {
         self.diagnostics.borrow_mut().push(diagnostic);
     }
 
-    pub fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator<Item = String>) {
+    #[allow(unused)]
+    pub(crate) fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator<Item = String>) {
         self.diagnostics.get_mut().extend(diagnostics);
     }
-
-    pub fn resolve_global_symbol(
-        &self,
-        module: &ModuleName,
-        symbol_name: &str,
-    ) -> QueryResult<Option<GlobalSymbolId>> {
-        let Some(module) = resolve_module(self.db.upcast(), module)? else {
-            return Ok(None);
-        };
-
-        resolve_global_symbol(self.db.upcast(), module, symbol_name)
-    }
 }
 
 #[derive(Debug)]
@@ -257,7 +203,7 @@ struct SyntaxLintVisitor<'a> {
 }
 
 impl Visitor<'_> for SyntaxLintVisitor<'_> {
-    fn visit_string_literal(&mut self, string_literal: &'_ StringLiteral) {
+    fn visit_string_literal(&mut self, string_literal: &'_ ast::StringLiteral) {
         // A very naive implementation of use double quotes
         let text = &self.source[string_literal.range];
 
@@ -268,10 +214,33 @@ impl Visitor<'_> for SyntaxLintVisitor<'_> {
     }
 }
 
-#[derive(Debug, Clone)]
+struct SemanticVisitor<'a> {
+    context: &'a SemanticLintContext<'a>,
+}
+
+impl Visitor<'_> for SemanticVisitor<'_> {
+    fn visit_stmt(&mut self, stmt: &ast::Stmt) {
+        match stmt {
+            ast::Stmt::ClassDef(class) => {
+                lint_bad_override(self.context, class);
+            }
+            ast::Stmt::Import(import) => {
+                lint_unresolved_imports(self.context, AnyImportRef::Import(import));
+            }
+            ast::Stmt::ImportFrom(import) => {
+                lint_unresolved_imports(self.context, AnyImportRef::ImportFrom(import));
+            }
+            _ => {}
+        }
+
+        walk_stmt(self, stmt);
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Diagnostics {
     Empty,
-    List(Arc<Vec<String>>),
+    List(Vec<String>),
 }
 
 impl Diagnostics {
@@ -295,41 +264,13 @@ impl From<Vec<String>> for Diagnostics {
         if value.is_empty() {
             Diagnostics::Empty
         } else {
-            Diagnostics::List(Arc::new(value))
+            Diagnostics::List(value)
         }
     }
 }
 
-#[derive(Default, Debug)]
+#[derive(Copy, Clone, Debug)]
-pub struct LintSyntaxStorage(KeyValueCache<FileId, Diagnostics>);
+enum AnyImportRef<'a> {
-
+    Import(&'a ast::StmtImport),
-impl Deref for LintSyntaxStorage {
+    ImportFrom(&'a ast::StmtImportFrom),
-    type Target = KeyValueCache<FileId, Diagnostics>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for LintSyntaxStorage {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-#[derive(Default, Debug)]
-pub struct LintSemanticStorage(KeyValueCache<FileId, Diagnostics>);
-
-impl Deref for LintSemanticStorage {
-    type Target = KeyValueCache<FileId, Diagnostics>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for LintSemanticStorage {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
 }

crates/red_knot/src/main.rs

@@ -1,9 +1,7 @@
-#![allow(clippy::dbg_macro)]
-
-use std::path::Path;
 use std::sync::Mutex;
 
 use crossbeam::channel as crossbeam_channel;
+use salsa::ParallelDatabase;
 use tracing::subscriber::Interest;
 use tracing::{Level, Metadata};
 use tracing_subscriber::filter::LevelFilter;
@@ -11,15 +9,21 @@ use tracing_subscriber::layer::{Context, Filter, SubscriberExt};
 use tracing_subscriber::{Layer, Registry};
 use tracing_tree::time::Uptime;
 
-use red_knot::db::{HasJar, ParallelDatabase, QueryError, SourceDb, SourceJar};
-use red_knot::module::{set_module_search_paths, ModuleResolutionInputs};
-use red_knot::program::check::ExecutionMode;
 use red_knot::program::{FileWatcherChange, Program};
 use red_knot::watch::FileWatcher;
 use red_knot::Workspace;
+use red_knot_module_resolver::{set_module_resolution_settings, ModuleResolutionSettings};
+use ruff_db::file_system::{FileSystem, FileSystemPath, OsFileSystem};
+use ruff_db::vfs::system_path_to_file;
 
-#[allow(clippy::print_stdout, clippy::unnecessary_wraps, clippy::print_stderr)]
+#[allow(
-fn main() -> anyhow::Result<()> {
+    clippy::print_stdout,
+    clippy::unnecessary_wraps,
+    clippy::print_stderr,
+    clippy::dbg_macro
+)]
+pub fn main() -> anyhow::Result<()> {
+    countme::enable(true);
     setup_tracing();
 
     let arguments: Vec<_> = std::env::args().collect();
@@ -29,34 +33,39 @@ fn main() -> anyhow::Result<()> {
         return Err(anyhow::anyhow!("Invalid arguments"));
     }
 
-    let entry_point = Path::new(&arguments[1]);
+    let fs = OsFileSystem;
+    let entry_point = FileSystemPath::new(&arguments[1]);
 
-    if !entry_point.exists() {
+    if !fs.exists(entry_point) {
         eprintln!("The entry point does not exist.");
         return Err(anyhow::anyhow!("Invalid arguments"));
     }
 
-    if !entry_point.is_file() {
+    if !fs.is_file(entry_point) {
         eprintln!("The entry point is not a file.");
         return Err(anyhow::anyhow!("Invalid arguments"));
     }
 
+    let entry_point = entry_point.to_path_buf();
+
     let workspace_folder = entry_point.parent().unwrap();
     let workspace = Workspace::new(workspace_folder.to_path_buf());
 
     let workspace_search_path = workspace.root().to_path_buf();
 
-    let search_paths = ModuleResolutionInputs {
+    let mut program = Program::new(workspace, fs);
+
+    set_module_resolution_settings(
+        &mut program,
+        ModuleResolutionSettings {
             extra_paths: vec![],
             workspace_root: workspace_search_path,
             site_packages: None,
             custom_typeshed: None,
-    };
+        },
+    );
 
-    let mut program = Program::new(workspace);
+    let entry_id = system_path_to_file(&program, entry_point.clone()).unwrap();
-    set_module_search_paths(&mut program, search_paths);
-
-    let entry_id = program.file_id(entry_point);
     program.workspace_mut().open_file(entry_id);
 
     let (main_loop, main_loop_cancellation_token) = MainLoop::new();
@@ -78,14 +87,11 @@ fn main() -> anyhow::Result<()> {
         file_changes_notifier.notify(changes);
     })?;
 
-    file_watcher.watch_folder(workspace_folder)?;
+    file_watcher.watch_folder(workspace_folder.as_std_path())?;
 
     main_loop.run(&mut program);
 
-    let source_jar: &SourceJar = program.jar().unwrap();
+    println!("{}", countme::get_all());
-
-    dbg!(source_jar.parsed.statistics());
-    dbg!(source_jar.sources.statistics());
 
     Ok(())
 }
@@ -127,6 +133,7 @@ impl MainLoop {
         }
     }
 
+    #[allow(clippy::print_stderr)]
     fn run(self, program: &mut Program) {
         self.orchestrator_sender
             .send(OrchestratorMessage::Run)
@@ -142,8 +149,8 @@ impl MainLoop {
 
                     // Spawn a new task that checks the program. This needs to be done in a separate thread
                     // to prevent blocking the main loop here.
-                    rayon::spawn(move || match program.check(ExecutionMode::ThreadPool) {
+                    rayon::spawn(move || {
-                        Ok(result) => {
+                        if let Ok(result) = program.check() {
                             sender
                                 .send(OrchestratorMessage::CheckProgramCompleted {
                                     diagnostics: result,
@@ -151,7 +158,6 @@ impl MainLoop {
                                 })
                                 .unwrap();
                         }
-                        Err(QueryError::Cancelled) => {}
                     });
                 }
                 MainLoopMessage::ApplyChanges(changes) => {
@@ -159,9 +165,11 @@ impl MainLoop {
                     program.apply_changes(changes);
                 }
                 MainLoopMessage::CheckCompleted(diagnostics) => {
-                    dbg!(diagnostics);
+                    eprintln!("{}", diagnostics.join("\n"));
+                    eprintln!("{}", countme::get_all());
                 }
                 MainLoopMessage::Exit => {
+                    eprintln!("{}", countme::get_all());
                     return;
                 }
             }
@@ -210,6 +218,7 @@ struct Orchestrator {
 }
 
 impl Orchestrator {
+    #[allow(clippy::print_stderr)]
     fn run(&mut self) {
         while let Ok(message) = self.receiver.recv() {
             match message {

crates/red_knot/src/parse.rs

@@ -1,41 +0,0 @@
-use std::ops::{Deref, DerefMut};
-use std::sync::Arc;
-
-use ruff_python_ast::ModModule;
-use ruff_python_parser::Parsed;
-
-use crate::cache::KeyValueCache;
-use crate::db::{QueryResult, SourceDb};
-use crate::files::FileId;
-use crate::source::source_text;
-
-#[tracing::instrument(level = "debug", skip(db))]
-pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Arc<Parsed<ModModule>>> {
-    let jar = db.jar()?;
-
-    jar.parsed.get(&file_id, |file_id| {
-        let source = source_text(db, *file_id)?;
-
-        Ok(Arc::new(ruff_python_parser::parse_unchecked_source(
-            source.text(),
-            source.kind().into(),
-        )))
-    })
-}
-
-#[derive(Debug, Default)]
-pub struct ParsedStorage(KeyValueCache<FileId, Arc<Parsed<ModModule>>>);
-
-impl Deref for ParsedStorage {
-    type Target = KeyValueCache<FileId, Arc<Parsed<ModModule>>>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for ParsedStorage {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}

crates/red_knot/src/program/check.rs

@@ -1,413 +1,28 @@
-use rayon::{current_num_threads, yield_local};
+use ruff_db::vfs::VfsFile;
-use rustc_hash::FxHashSet;
+use salsa::Cancelled;
 
-use crate::db::{Database, QueryError, QueryResult};
-use crate::files::FileId;
 use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
-use crate::module::{file_to_module, resolve_module};
 use crate::program::Program;
-use crate::semantic::{semantic_index, Dependency};
 
 impl Program {
     /// Checks all open files in the workspace and its dependencies.
     #[tracing::instrument(level = "debug", skip_all)]
-    pub fn check(&self, mode: ExecutionMode) -> QueryResult<Vec<String>> {
+    pub fn check(&self) -> Result<Vec<String>, Cancelled> {
-        self.cancelled()?;
+        self.with_db(|db| {
-
+            let mut result = Vec::new();
-        let mut context = CheckContext::new(self);
+            for open_file in db.workspace.open_files() {
-
+                result.extend_from_slice(&db.check_file(open_file));
-        match mode {
-            ExecutionMode::SingleThreaded => SingleThreadedExecutor.run(&mut context)?,
-            ExecutionMode::ThreadPool => ThreadPoolExecutor.run(&mut context)?,
-        };
-
-        Ok(context.finish())
             }
 
-    #[tracing::instrument(level = "debug", skip(self, context))]
-    fn check_file(&self, file: FileId, context: &CheckFileContext) -> QueryResult<Diagnostics> {
-        self.cancelled()?;
-
-        let index = semantic_index(self, file)?;
-        let dependencies = index.symbol_table().dependencies();
-
-        if !dependencies.is_empty() {
-            let module = file_to_module(self, file)?;
-
-            // TODO scheduling all dependencies here is wasteful if we don't infer any types on them
-            //  but I think that's unlikely, so it is okay?
-            //  Anyway, we need to figure out a way to retrieve the dependencies of a module
-            //  from the persistent cache. So maybe it should be a separate query after all.
-            for dependency in dependencies {
-                let dependency_name = match dependency {
-                    Dependency::Module(name) => Some(name.clone()),
-                    Dependency::Relative { .. } => match &module {
-                        Some(module) => module.resolve_dependency(self, dependency)?,
-                        None => None,
-                    },
-                };
-
-                if let Some(dependency_name) = dependency_name {
-                    // TODO We may want to have a different check functions for non-first-party
-                    //   files because we only need to index them and not check them.
-                    //   Supporting non-first-party code also requires supporting typing stubs.
-                    if let Some(dependency) = resolve_module(self, &dependency_name)? {
-                        if dependency.path(self)?.root().kind().is_first_party() {
-                            context.schedule_dependency(dependency.path(self)?.file());
-                        }
-                    }
-                }
-            }
-        }
-
-        let mut diagnostics = Vec::new();
-
-        if self.workspace().is_file_open(file) {
-            diagnostics.extend_from_slice(&lint_syntax(self, file)?);
-            diagnostics.extend_from_slice(&lint_semantic(self, file)?);
-        }
-
-        Ok(Diagnostics::from(diagnostics))
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum ExecutionMode {
-    SingleThreaded,
-    ThreadPool,
-}
-
-/// Context that stores state information about the entire check operation.
-struct CheckContext<'a> {
-    /// IDs of the files that have been queued for checking.
-    ///
-    /// Used to avoid queuing the same file twice.
-    scheduled_files: FxHashSet<FileId>,
-
-    /// Reference to the program that is checked.
-    program: &'a Program,
-
-    /// The aggregated diagnostics
-    diagnostics: Vec<String>,
-}
-
-impl<'a> CheckContext<'a> {
-    fn new(program: &'a Program) -> Self {
-        Self {
-            scheduled_files: FxHashSet::default(),
-            program,
-            diagnostics: Vec::new(),
-        }
-    }
-
-    /// Returns the tasks to check all open files in the workspace.
-    fn check_open_files(&mut self) -> Vec<CheckOpenFileTask> {
-        self.scheduled_files
-            .extend(self.program.workspace().open_files());
-
-        self.program
-            .workspace()
-            .open_files()
-            .map(|file_id| CheckOpenFileTask { file_id })
-            .collect()
-    }
-
-    /// Returns the task to check a dependency.
-    fn check_dependency(&mut self, file_id: FileId) -> Option<CheckDependencyTask> {
-        if self.scheduled_files.insert(file_id) {
-            Some(CheckDependencyTask { file_id })
-        } else {
-            None
-        }
-    }
-
-    /// Pushes the result for a single file check operation
-    fn push_diagnostics(&mut self, diagnostics: &Diagnostics) {
-        self.diagnostics.extend_from_slice(diagnostics);
-    }
-
-    /// Returns a reference to the program that is being checked.
-    fn program(&self) -> &'a Program {
-        self.program
-    }
-
-    /// Creates a task context that is used to check a single file.
-    fn task_context<'b, S>(&self, dependency_scheduler: &'b S) -> CheckTaskContext<'a, 'b, S>
-    where
-        S: ScheduleDependency,
-    {
-        CheckTaskContext {
-            program: self.program,
-            dependency_scheduler,
-        }
-    }
-
-    fn finish(self) -> Vec<String> {
-        self.diagnostics
-    }
-}
-
-/// Trait that abstracts away how a dependency of a file gets scheduled for checking.
-trait ScheduleDependency {
-    /// Schedules the file with the given ID for checking.
-    fn schedule(&self, file_id: FileId);
-}
-
-impl<T> ScheduleDependency for T
-where
-    T: Fn(FileId),
-{
-    fn schedule(&self, file_id: FileId) {
-        let f = self;
-        f(file_id);
-    }
-}
-
-/// Context that is used to run a single file check task.
-///
-/// The task is generic over `S` because it is passed across thread boundaries and
-/// we don't want to add the requirement that [`ScheduleDependency`] must be [`Send`].
-struct CheckTaskContext<'a, 'scheduler, S>
-where
-    S: ScheduleDependency,
-{
-    dependency_scheduler: &'scheduler S,
-    program: &'a Program,
-}
-
-impl<'a, 'scheduler, S> CheckTaskContext<'a, 'scheduler, S>
-where
-    S: ScheduleDependency,
-{
-    fn as_file_context(&self) -> CheckFileContext<'scheduler> {
-        CheckFileContext {
-            dependency_scheduler: self.dependency_scheduler,
-        }
-    }
-}
-
-/// Context passed when checking a single file.
-///
-/// This is a trimmed down version of [`CheckTaskContext`] with the type parameter `S` erased
-/// to avoid monomorphization of [`Program:check_file`].
-struct CheckFileContext<'a> {
-    dependency_scheduler: &'a dyn ScheduleDependency,
-}
-
-impl<'a> CheckFileContext<'a> {
-    fn schedule_dependency(&self, file_id: FileId) {
-        self.dependency_scheduler.schedule(file_id);
-    }
-}
-
-#[derive(Debug)]
-enum CheckFileTask {
-    OpenFile(CheckOpenFileTask),
-    Dependency(CheckDependencyTask),
-}
-
-impl CheckFileTask {
-    /// Runs the task and returns the results for checking this file.
-    fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
-    where
-        S: ScheduleDependency,
-    {
-        match self {
-            Self::OpenFile(task) => task.run(context),
-            Self::Dependency(task) => task.run(context),
-        }
-    }
-
-    fn file_id(&self) -> FileId {
-        match self {
-            CheckFileTask::OpenFile(task) => task.file_id,
-            CheckFileTask::Dependency(task) => task.file_id,
-        }
-    }
-}
-
-/// Task to check an open file.
-
-#[derive(Debug)]
-struct CheckOpenFileTask {
-    file_id: FileId,
-}
-
-impl CheckOpenFileTask {
-    fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
-    where
-        S: ScheduleDependency,
-    {
-        context
-            .program
-            .check_file(self.file_id, &context.as_file_context())
-    }
-}
-
-/// Task to check a dependency file.
-#[derive(Debug)]
-struct CheckDependencyTask {
-    file_id: FileId,
-}
-
-impl CheckDependencyTask {
-    fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
-    where
-        S: ScheduleDependency,
-    {
-        context
-            .program
-            .check_file(self.file_id, &context.as_file_context())
-    }
-}
-
-/// Executor that schedules the checking of individual program files.
-trait CheckExecutor {
-    fn run(self, context: &mut CheckContext) -> QueryResult<()>;
-}
-
-/// Executor that runs all check operations on the current thread.
-///
-/// The executor does not schedule dependencies for checking.
-/// The main motivation for scheduling dependencies
-/// in a multithreaded environment is to parse and index the dependencies concurrently.
-/// However, that doesn't make sense in a single threaded environment, because the dependencies then compute
-/// with checking the open files. Checking dependencies in a single threaded environment is more likely
-/// to hurt performance because we end up analyzing files in their entirety, even if we only need to type check parts of them.
-#[derive(Debug, Default)]
-struct SingleThreadedExecutor;
-
-impl CheckExecutor for SingleThreadedExecutor {
-    fn run(self, context: &mut CheckContext) -> QueryResult<()> {
-        let mut queue = context.check_open_files();
-
-        let noop_schedule_dependency = |_| {};
-
-        while let Some(file) = queue.pop() {
-            context.program().cancelled()?;
-
-            let task_context = context.task_context(&noop_schedule_dependency);
-            context.push_diagnostics(&file.run(&task_context)?);
-        }
-
-        Ok(())
-    }
-}
-
-/// Executor that runs the check operations on a thread pool.
-///
-/// The executor runs each check operation as its own task using a thread pool.
-///
-/// Other than [`SingleThreadedExecutor`], this executor schedules dependencies for checking. It
-/// even schedules dependencies for checking when the thread pool size is 1 for a better debugging experience.
-#[derive(Debug, Default)]
-struct ThreadPoolExecutor;
-
-impl CheckExecutor for ThreadPoolExecutor {
-    fn run(self, context: &mut CheckContext) -> QueryResult<()> {
-        let num_threads = current_num_threads();
-        let single_threaded = num_threads == 1;
-        let span = tracing::trace_span!("ThreadPoolExecutor::run", num_threads);
-        let _ = span.enter();
-
-        let mut queue: Vec<_> = context
-            .check_open_files()
-            .into_iter()
-            .map(CheckFileTask::OpenFile)
-            .collect();
-
-        let (sender, receiver) = if single_threaded {
-            // Use an unbounded queue for single threaded execution to prevent deadlocks
-            // when a single file schedules multiple dependencies.
-            crossbeam::channel::unbounded()
-        } else {
-            // Use a bounded queue to apply backpressure when the orchestration thread isn't able to keep
-            // up processing messages from the worker threads.
-            crossbeam::channel::bounded(num_threads)
-        };
-
-        let schedule_sender = sender.clone();
-        let schedule_dependency = move |file_id| {
-            schedule_sender
-                .send(ThreadPoolMessage::ScheduleDependency(file_id))
-                .unwrap();
-        };
-
-        let result = rayon::in_place_scope(|scope| {
-            let mut pending = 0usize;
-
-            loop {
-                context.program().cancelled()?;
-
-                // 1. Try to get a queued message to ensure that we have always remaining space in the channel to prevent blocking the worker threads.
-                // 2. Try to process a queued file
-                // 3. If there's no queued file wait for the next incoming message.
-                // 4. Exit if there are no more messages and no senders.
-                let message = if let Ok(message) = receiver.try_recv() {
-                    message
-                } else if let Some(task) = queue.pop() {
-                    pending += 1;
-
-                    let task_context = context.task_context(&schedule_dependency);
-                    let sender = sender.clone();
-                    let task_span = tracing::trace_span!(
-                        parent: &span,
-                        "CheckFileTask::run",
-                        file_id = task.file_id().as_u32(),
-                    );
-
-                    scope.spawn(move |_| {
-                        task_span.in_scope(|| match task.run(&task_context) {
-                            Ok(result) => {
-                                sender.send(ThreadPoolMessage::Completed(result)).unwrap();
-                            }
-                            Err(err) => sender.send(ThreadPoolMessage::Errored(err)).unwrap(),
-                        });
-                    });
-
-                    // If this is a single threaded rayon thread pool, yield the current thread
-                    // or we never start processing the work items.
-                    if single_threaded {
-                        yield_local();
-                    }
-
-                    continue;
-                } else if let Ok(message) = receiver.recv() {
-                    message
-                } else {
-                    break;
-                };
-
-                match message {
-                    ThreadPoolMessage::ScheduleDependency(dependency) => {
-                        if let Some(task) = context.check_dependency(dependency) {
-                            queue.push(CheckFileTask::Dependency(task));
-                        }
-                    }
-                    ThreadPoolMessage::Completed(diagnostics) => {
-                        context.push_diagnostics(&diagnostics);
-                        pending -= 1;
-
-                        if pending == 0 && queue.is_empty() {
-                            break;
-                        }
-                    }
-                    ThreadPoolMessage::Errored(err) => {
-                        return Err(err);
-                    }
-                }
-            }
-
-            Ok(())
-        });
-
             result
-    }
+        })
     }
 
-#[derive(Debug)]
+    #[tracing::instrument(level = "debug", skip(self))]
-enum ThreadPoolMessage {
+    fn check_file(&self, file: VfsFile) -> Diagnostics {
-    ScheduleDependency(FileId),
+        let mut diagnostics = Vec::new();
-    Completed(Diagnostics),
+        diagnostics.extend_from_slice(lint_syntax(self, file));
-    Errored(QueryError),
+        diagnostics.extend_from_slice(lint_semantic(self, file));
+        Diagnostics::from(diagnostics)
+    }
 }

crates/red_knot/src/program/mod.rs

@@ -1,30 +1,36 @@
-use std::collections::hash_map::Entry;
+use std::panic::{RefUnwindSafe, UnwindSafe};
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use rustc_hash::FxHashMap;
+use salsa::{Cancelled, Database};
 
-use crate::db::{
+use red_knot_module_resolver::{Db as ResolverDb, Jar as ResolverJar};
-    Database, Db, DbRuntime, DbWithJar, HasJar, HasJars, JarsStorage, LintDb, LintJar,
+use red_knot_python_semantic::{Db as SemanticDb, Jar as SemanticJar};
-    ParallelDatabase, QueryResult, SemanticDb, SemanticJar, Snapshot, SourceDb, SourceJar, Upcast,
+use ruff_db::file_system::{FileSystem, FileSystemPathBuf};
-};
+use ruff_db::vfs::{Vfs, VfsFile, VfsPath};
-use crate::files::{FileId, Files};
+use ruff_db::{Db as SourceDb, Jar as SourceJar, Upcast};
+
+use crate::db::{Db, Jar};
 use crate::Workspace;
 
-pub mod check;
+mod check;
 
-#[derive(Debug)]
+#[salsa::db(SourceJar, ResolverJar, SemanticJar, Jar)]
 pub struct Program {
-    jars: JarsStorage<Program>,
+    storage: salsa::Storage<Program>,
-    files: Files,
+    vfs: Vfs,
+    fs: Arc<dyn FileSystem + Send + Sync + RefUnwindSafe>,
     workspace: Workspace,
 }
 
 impl Program {
-    pub fn new(workspace: Workspace) -> Self {
+    pub fn new<Fs>(workspace: Workspace, file_system: Fs) -> Self
+    where
+        Fs: FileSystem + 'static + Send + Sync + RefUnwindSafe,
+    {
         Self {
-            jars: JarsStorage::default(),
+            storage: salsa::Storage::default(),
-            files: Files::default(),
+            vfs: Vfs::default(),
+            fs: Arc::new(file_system),
             workspace,
         }
     }
@@ -33,30 +39,11 @@ impl Program {
     where
         I: IntoIterator<Item = FileWatcherChange>,
     {
-        let mut aggregated_changes = AggregatedChanges::default();
+        for change in changes {
-
+            VfsFile::touch_path(self, &VfsPath::file_system(change.path));
-        aggregated_changes.extend(changes.into_iter().map(|change| FileChange {
-            id: self.files.intern(&change.path),
-            kind: change.kind,
-        }));
-
-        let (source, semantic, lint) = self.jars_mut();
-        for change in aggregated_changes.iter() {
-            semantic.module_resolver.remove_module_by_file(change.id);
-            semantic.semantic_indices.remove(&change.id);
-            source.sources.remove(&change.id);
-            source.parsed.remove(&change.id);
-            // TODO: remove all dependent modules as well
-            semantic.type_store.remove_module(change.id);
-            lint.lint_syntax.remove(&change.id);
-            lint.lint_semantic.remove(&change.id);
         }
     }
 
-    pub fn files(&self) -> &Files {
-        &self.files
-    }
-
     pub fn workspace(&self) -> &Workspace {
         &self.workspace
     }
@@ -64,28 +51,18 @@ impl Program {
     pub fn workspace_mut(&mut self) -> &mut Workspace {
         &mut self.workspace
     }
-}
 
-impl SourceDb for Program {
+    #[allow(clippy::unnecessary_wraps)]
-    fn file_id(&self, path: &Path) -> FileId {
+    fn with_db<F, T>(&self, f: F) -> Result<T, Cancelled>
-        self.files.intern(path)
+    where
-    }
+        F: FnOnce(&Program) -> T + UnwindSafe,
-
+    {
-    fn file_path(&self, file_id: FileId) -> Arc<Path> {
+        // TODO: Catch in `Caancelled::catch`
-        self.files.path(file_id)
+        //  See https://salsa.zulipchat.com/#narrow/stream/145099-general/topic/How.20to.20use.20.60Cancelled.3A.3Acatch.60
+        Ok(f(self))
     }
 }
 
-impl DbWithJar<SourceJar> for Program {}
-
-impl SemanticDb for Program {}
-
-impl DbWithJar<SemanticJar> for Program {}
-
-impl LintDb for Program {}
-
-impl DbWithJar<LintJar> for Program {}
-
 impl Upcast<dyn SemanticDb> for Program {
     fn upcast(&self) -> &(dyn SemanticDb + 'static) {
         self
@@ -98,178 +75,57 @@ impl Upcast<dyn SourceDb> for Program {
     }
 }
 
-impl Upcast<dyn LintDb> for Program {
+impl Upcast<dyn ResolverDb> for Program {
-    fn upcast(&self) -> &(dyn LintDb + 'static) {
+    fn upcast(&self) -> &(dyn ResolverDb + 'static) {
         self
     }
 }
 
+impl ResolverDb for Program {}
+
+impl SemanticDb for Program {}
+
+impl SourceDb for Program {
+    fn file_system(&self) -> &dyn FileSystem {
+        &*self.fs
+    }
+
+    fn vfs(&self) -> &Vfs {
+        &self.vfs
+    }
+}
+
+impl Database for Program {}
+
 impl Db for Program {}
 
-impl Database for Program {
+impl salsa::ParallelDatabase for Program {
-    fn runtime(&self) -> &DbRuntime {
+    fn snapshot(&self) -> salsa::Snapshot<Self> {
-        self.jars.runtime()
+        salsa::Snapshot::new(Self {
-    }
+            storage: self.storage.snapshot(),
-
+            vfs: self.vfs.snapshot(),
-    fn runtime_mut(&mut self) -> &mut DbRuntime {
+            fs: self.fs.clone(),
-        self.jars.runtime_mut()
-    }
-}
-
-impl ParallelDatabase for Program {
-    fn snapshot(&self) -> Snapshot<Self> {
-        Snapshot::new(Self {
-            jars: self.jars.snapshot(),
-            files: self.files.snapshot(),
             workspace: self.workspace.clone(),
         })
     }
 }
 
-impl HasJars for Program {
-    type Jars = (SourceJar, SemanticJar, LintJar);
-
-    fn jars(&self) -> QueryResult<&Self::Jars> {
-        self.jars.jars()
-    }
-
-    fn jars_mut(&mut self) -> &mut Self::Jars {
-        self.jars.jars_mut()
-    }
-}
-
-impl HasJar<SourceJar> for Program {
-    fn jar(&self) -> QueryResult<&SourceJar> {
-        Ok(&self.jars()?.0)
-    }
-
-    fn jar_mut(&mut self) -> &mut SourceJar {
-        &mut self.jars_mut().0
-    }
-}
-
-impl HasJar<SemanticJar> for Program {
-    fn jar(&self) -> QueryResult<&SemanticJar> {
-        Ok(&self.jars()?.1)
-    }
-
-    fn jar_mut(&mut self) -> &mut SemanticJar {
-        &mut self.jars_mut().1
-    }
-}
-
-impl HasJar<LintJar> for Program {
-    fn jar(&self) -> QueryResult<&LintJar> {
-        Ok(&self.jars()?.2)
-    }
-
-    fn jar_mut(&mut self) -> &mut LintJar {
-        &mut self.jars_mut().2
-    }
-}
-
 #[derive(Clone, Debug)]
 pub struct FileWatcherChange {
-    path: PathBuf,
+    path: FileSystemPathBuf,
+    #[allow(unused)]
     kind: FileChangeKind,
 }
 
 impl FileWatcherChange {
-    pub fn new(path: PathBuf, kind: FileChangeKind) -> Self {
+    pub fn new(path: FileSystemPathBuf, kind: FileChangeKind) -> Self {
         Self { path, kind }
     }
 }
 
-#[derive(Copy, Clone, Debug)]
-struct FileChange {
-    id: FileId,
-    kind: FileChangeKind,
-}
-
-impl FileChange {
-    fn file_id(self) -> FileId {
-        self.id
-    }
-
-    fn kind(self) -> FileChangeKind {
-        self.kind
-    }
-}
-
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum FileChangeKind {
     Created,
     Modified,
     Deleted,
 }
-
-#[derive(Default, Debug)]
-struct AggregatedChanges {
-    changes: FxHashMap<FileId, FileChangeKind>,
-}
-
-impl AggregatedChanges {
-    fn add(&mut self, change: FileChange) {
-        match self.changes.entry(change.file_id()) {
-            Entry::Occupied(mut entry) => {
-                let merged = entry.get_mut();
-
-                match (merged, change.kind()) {
-                    (FileChangeKind::Created, FileChangeKind::Deleted) => {
-                        // Deletion after creations means that ruff never saw the file.
-                        entry.remove();
-                    }
-                    (FileChangeKind::Created, FileChangeKind::Modified) => {
-                        // No-op, for ruff, modifying a file that it doesn't yet know that it exists is still considered a creation.
-                    }
-
-                    (FileChangeKind::Modified, FileChangeKind::Created) => {
-                        // Uhh, that should probably not happen. Continue considering it a modification.
-                    }
-
-                    (FileChangeKind::Modified, FileChangeKind::Deleted) => {
-                        *entry.get_mut() = FileChangeKind::Deleted;
-                    }
-
-                    (FileChangeKind::Deleted, FileChangeKind::Created) => {
-                        *entry.get_mut() = FileChangeKind::Modified;
-                    }
-
-                    (FileChangeKind::Deleted, FileChangeKind::Modified) => {
-                        // That's weird, but let's consider it a modification.
-                        *entry.get_mut() = FileChangeKind::Modified;
-                    }
-
-                    (FileChangeKind::Created, FileChangeKind::Created)
-                    | (FileChangeKind::Modified, FileChangeKind::Modified)
-                    | (FileChangeKind::Deleted, FileChangeKind::Deleted) => {
-                        // No-op transitions. Some of them should be impossible but we handle them anyway.
-                    }
-                }
-            }
-            Entry::Vacant(entry) => {
-                entry.insert(change.kind());
-            }
-        }
-    }
-
-    fn extend<I>(&mut self, changes: I)
-    where
-        I: IntoIterator<Item = FileChange>,
-    {
-        let iter = changes.into_iter();
-        let (lower, _) = iter.size_hint();
-        self.changes.reserve(lower);
-
-        for change in iter {
-            self.add(change);
-        }
-    }
-
-    fn iter(&self) -> impl Iterator<Item = FileChange> + '_ {
-        self.changes.iter().map(|(id, kind)| FileChange {
-            id: *id,
-            kind: *kind,
-        })
-    }
-}

crates/red_knot/src/semantic.rs

@@ -1,881 +0,0 @@
-use std::num::NonZeroU32;
-
-use ruff_python_ast as ast;
-use ruff_python_ast::visitor::source_order::SourceOrderVisitor;
-use ruff_python_ast::AstNode;
-
-use crate::ast_ids::{NodeKey, TypedNodeKey};
-use crate::cache::KeyValueCache;
-use crate::db::{QueryResult, SemanticDb, SemanticJar};
-use crate::files::FileId;
-use crate::module::Module;
-use crate::parse::parse;
-pub(crate) use definitions::Definition;
-use definitions::{ImportDefinition, ImportFromDefinition};
-pub(crate) use flow_graph::ConstrainedDefinition;
-use flow_graph::{FlowGraph, FlowGraphBuilder, FlowNodeId, ReachableDefinitionsIterator};
-use red_knot_module_resolver::ModuleName;
-use ruff_index::{newtype_index, IndexVec};
-use rustc_hash::FxHashMap;
-use std::ops::{Deref, DerefMut};
-use std::sync::Arc;
-pub(crate) use symbol_table::{Dependency, SymbolId};
-use symbol_table::{ScopeId, ScopeKind, SymbolFlags, SymbolTable, SymbolTableBuilder};
-pub(crate) use types::{infer_definition_type, infer_symbol_public_type, Type, TypeStore};
-
-mod definitions;
-mod flow_graph;
-mod symbol_table;
-mod types;
-
-#[tracing::instrument(level = "debug", skip(db))]
-pub fn semantic_index(db: &dyn SemanticDb, file_id: FileId) -> QueryResult<Arc<SemanticIndex>> {
-    let jar: &SemanticJar = db.jar()?;
-
-    jar.semantic_indices.get(&file_id, |_| {
-        let parsed = parse(db.upcast(), file_id)?;
-        Ok(Arc::from(SemanticIndex::from_ast(parsed.syntax())))
-    })
-}
-
-#[tracing::instrument(level = "debug", skip(db))]
-pub fn resolve_global_symbol(
-    db: &dyn SemanticDb,
-    module: Module,
-    name: &str,
-) -> QueryResult<Option<GlobalSymbolId>> {
-    let file_id = module.path(db)?.file();
-    let symbol_table = &semantic_index(db, file_id)?.symbol_table;
-    let Some(symbol_id) = symbol_table.root_symbol_id_by_name(name) else {
-        return Ok(None);
-    };
-    Ok(Some(GlobalSymbolId { file_id, symbol_id }))
-}
-
-#[newtype_index]
-pub struct ExpressionId;
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub struct GlobalSymbolId {
-    pub(crate) file_id: FileId,
-    pub(crate) symbol_id: SymbolId,
-}
-
-#[derive(Debug)]
-pub struct SemanticIndex {
-    symbol_table: SymbolTable,
-    flow_graph: FlowGraph,
-    expressions: FxHashMap<NodeKey, ExpressionId>,
-    expressions_by_id: IndexVec<ExpressionId, NodeKey>,
-}
-
-impl SemanticIndex {
-    pub fn from_ast(module: &ast::ModModule) -> Self {
-        let root_scope_id = SymbolTable::root_scope_id();
-        let mut indexer = SemanticIndexer {
-            symbol_table_builder: SymbolTableBuilder::new(),
-            flow_graph_builder: FlowGraphBuilder::new(),
-            scopes: vec![ScopeState {
-                scope_id: root_scope_id,
-                current_flow_node_id: FlowGraph::start(),
-            }],
-            expressions: FxHashMap::default(),
-            expressions_by_id: IndexVec::default(),
-            current_definition: None,
-        };
-        indexer.visit_body(&module.body);
-        indexer.finish()
-    }
-
-    fn resolve_expression_id<'a>(
-        &self,
-        ast: &'a ast::ModModule,
-        expression_id: ExpressionId,
-    ) -> ast::AnyNodeRef<'a> {
-        let node_key = self.expressions_by_id[expression_id];
-        node_key
-            .resolve(ast.as_any_node_ref())
-            .expect("node to resolve")
-    }
-
-    /// Return an iterator over all definitions of `symbol_id` reachable from `use_expr`. The value
-    /// of `symbol_id` in `use_expr` must originate from one of the iterated definitions (or from
-    /// an external reassignment of the name outside of this scope).
-    pub fn reachable_definitions(
-        &self,
-        symbol_id: SymbolId,
-        use_expr: &ast::Expr,
-    ) -> ReachableDefinitionsIterator {
-        let expression_id = self.expression_id(use_expr);
-        ReachableDefinitionsIterator::new(
-            &self.flow_graph,
-            symbol_id,
-            self.flow_graph.for_expr(expression_id),
-        )
-    }
-
-    pub fn expression_id(&self, expression: &ast::Expr) -> ExpressionId {
-        self.expressions[&NodeKey::from_node(expression.into())]
-    }
-
-    pub fn symbol_table(&self) -> &SymbolTable {
-        &self.symbol_table
-    }
-}
-
-#[derive(Debug)]
-struct ScopeState {
-    scope_id: ScopeId,
-    current_flow_node_id: FlowNodeId,
-}
-
-#[derive(Debug)]
-struct SemanticIndexer {
-    symbol_table_builder: SymbolTableBuilder,
-    flow_graph_builder: FlowGraphBuilder,
-    scopes: Vec<ScopeState>,
-    /// the definition whose target(s) we are currently walking
-    current_definition: Option<Definition>,
-    expressions: FxHashMap<NodeKey, ExpressionId>,
-    expressions_by_id: IndexVec<ExpressionId, NodeKey>,
-}
-
-impl SemanticIndexer {
-    pub(crate) fn finish(mut self) -> SemanticIndex {
-        let SemanticIndexer {
-            flow_graph_builder,
-            symbol_table_builder,
-            ..
-        } = self;
-        self.expressions.shrink_to_fit();
-        self.expressions_by_id.shrink_to_fit();
-        SemanticIndex {
-            flow_graph: flow_graph_builder.finish(),
-            symbol_table: symbol_table_builder.finish(),
-            expressions: self.expressions,
-            expressions_by_id: self.expressions_by_id,
-        }
-    }
-
-    fn set_current_flow_node(&mut self, new_flow_node_id: FlowNodeId) {
-        let scope_state = self.scopes.last_mut().expect("scope stack is never empty");
-        scope_state.current_flow_node_id = new_flow_node_id;
-    }
-
-    fn current_flow_node(&self) -> FlowNodeId {
-        self.scopes
-            .last()
-            .expect("scope stack is never empty")
-            .current_flow_node_id
-    }
-
-    fn add_or_update_symbol(&mut self, identifier: &str, flags: SymbolFlags) -> SymbolId {
-        self.symbol_table_builder
-            .add_or_update_symbol(self.cur_scope(), identifier, flags)
-    }
-
-    fn add_or_update_symbol_with_def(
-        &mut self,
-        identifier: &str,
-        definition: Definition,
-    ) -> SymbolId {
-        let symbol_id = self.add_or_update_symbol(identifier, SymbolFlags::IS_DEFINED);
-        self.symbol_table_builder
-            .add_definition(symbol_id, definition.clone());
-        let new_flow_node_id =
-            self.flow_graph_builder
-                .add_definition(symbol_id, definition, self.current_flow_node());
-        self.set_current_flow_node(new_flow_node_id);
-        symbol_id
-    }
-
-    fn push_scope(
-        &mut self,
-        name: &str,
-        kind: ScopeKind,
-        definition: Option<Definition>,
-        defining_symbol: Option<SymbolId>,
-    ) -> ScopeId {
-        let scope_id = self.symbol_table_builder.add_child_scope(
-            self.cur_scope(),
-            name,
-            kind,
-            definition,
-            defining_symbol,
-        );
-        self.scopes.push(ScopeState {
-            scope_id,
-            current_flow_node_id: FlowGraph::start(),
-        });
-        scope_id
-    }
-
-    fn pop_scope(&mut self) -> ScopeId {
-        self.scopes
-            .pop()
-            .expect("Scope stack should never be empty")
-            .scope_id
-    }
-
-    fn cur_scope(&self) -> ScopeId {
-        self.scopes
-            .last()
-            .expect("Scope stack should never be empty")
-            .scope_id
-    }
-
-    fn record_scope_for_node(&mut self, node_key: NodeKey, scope_id: ScopeId) {
-        self.symbol_table_builder
-            .record_scope_for_node(node_key, scope_id);
-    }
-
-    fn insert_constraint(&mut self, expr: &ast::Expr) {
-        let node_key = NodeKey::from_node(expr.into());
-        let expression_id = self.expressions[&node_key];
-        let constraint = self
-            .flow_graph_builder
-            .add_constraint(self.current_flow_node(), expression_id);
-        self.set_current_flow_node(constraint);
-    }
-
-    fn with_type_params(
-        &mut self,
-        name: &str,
-        params: &Option<Box<ast::TypeParams>>,
-        definition: Option<Definition>,
-        defining_symbol: Option<SymbolId>,
-        nested: impl FnOnce(&mut Self) -> ScopeId,
-    ) -> ScopeId {
-        if let Some(type_params) = params {
-            self.push_scope(name, ScopeKind::Annotation, definition, defining_symbol);
-            for type_param in &type_params.type_params {
-                let name = match type_param {
-                    ast::TypeParam::TypeVar(ast::TypeParamTypeVar { name, .. }) => name,
-                    ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { name, .. }) => name,
-                    ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { name, .. }) => name,
-                };
-                self.add_or_update_symbol(name, SymbolFlags::IS_DEFINED);
-            }
-        }
-        let scope_id = nested(self);
-        if params.is_some() {
-            self.pop_scope();
-        }
-        scope_id
-    }
-}
-
-impl SourceOrderVisitor<'_> for SemanticIndexer {
-    fn visit_expr(&mut self, expr: &ast::Expr) {
-        let node_key = NodeKey::from_node(expr.into());
-        let expression_id = self.expressions_by_id.push(node_key);
-
-        let flow_expression_id = self
-            .flow_graph_builder
-            .record_expr(self.current_flow_node());
-        debug_assert_eq!(expression_id, flow_expression_id);
-
-        let symbol_expression_id = self
-            .symbol_table_builder
-            .record_expression(self.cur_scope());
-
-        debug_assert_eq!(expression_id, symbol_expression_id);
-
-        self.expressions.insert(node_key, expression_id);
-
-        match expr {
-            ast::Expr::Name(ast::ExprName { id, ctx, .. }) => {
-                let flags = match ctx {
-                    ast::ExprContext::Load => SymbolFlags::IS_USED,
-                    ast::ExprContext::Store => SymbolFlags::IS_DEFINED,
-                    ast::ExprContext::Del => SymbolFlags::IS_DEFINED,
-                    ast::ExprContext::Invalid => SymbolFlags::empty(),
-                };
-                self.add_or_update_symbol(id, flags);
-                if flags.contains(SymbolFlags::IS_DEFINED) {
-                    if let Some(curdef) = self.current_definition.clone() {
-                        self.add_or_update_symbol_with_def(id, curdef);
-                    }
-                }
-                ast::visitor::source_order::walk_expr(self, expr);
-            }
-            ast::Expr::Named(node) => {
-                debug_assert!(self.current_definition.is_none());
-                self.current_definition =
-                    Some(Definition::NamedExpr(TypedNodeKey::from_node(node)));
-                // TODO walrus in comprehensions is implicitly nonlocal
-                self.visit_expr(&node.target);
-                self.current_definition = None;
-                self.visit_expr(&node.value);
-            }
-            ast::Expr::If(ast::ExprIf {
-                body, test, orelse, ..
-            }) => {
-                // TODO detect statically known truthy or falsy test (via type inference, not naive
-                // AST inspection, so we can't simplify here, need to record test expression in CFG
-                // for later checking)
-
-                self.visit_expr(test);
-
-                let if_branch = self.flow_graph_builder.add_branch(self.current_flow_node());
-
-                self.set_current_flow_node(if_branch);
-                self.insert_constraint(test);
-                self.visit_expr(body);
-
-                let post_body = self.current_flow_node();
-
-                self.set_current_flow_node(if_branch);
-                self.visit_expr(orelse);
-
-                let post_else = self
-                    .flow_graph_builder
-                    .add_phi(self.current_flow_node(), post_body);
-
-                self.set_current_flow_node(post_else);
-            }
-            _ => {
-                ast::visitor::source_order::walk_expr(self, expr);
-            }
-        }
-    }
-
-    fn visit_stmt(&mut self, stmt: &ast::Stmt) {
-        // TODO need to capture more definition statements here
-        match stmt {
-            ast::Stmt::ClassDef(node) => {
-                let node_key = TypedNodeKey::from_node(node);
-                let def = Definition::ClassDef(node_key.clone());
-                let symbol_id = self.add_or_update_symbol_with_def(&node.name, def.clone());
-                for decorator in &node.decorator_list {
-                    self.visit_decorator(decorator);
-                }
-                let scope_id = self.with_type_params(
-                    &node.name,
-                    &node.type_params,
-                    Some(def.clone()),
-                    Some(symbol_id),
-                    |indexer| {
-                        if let Some(arguments) = &node.arguments {
-                            indexer.visit_arguments(arguments);
-                        }
-                        let scope_id = indexer.push_scope(
-                            &node.name,
-                            ScopeKind::Class,
-                            Some(def.clone()),
-                            Some(symbol_id),
-                        );
-                        indexer.visit_body(&node.body);
-                        indexer.pop_scope();
-                        scope_id
-                    },
-                );
-                self.record_scope_for_node(*node_key.erased(), scope_id);
-            }
-            ast::Stmt::FunctionDef(node) => {
-                let node_key = TypedNodeKey::from_node(node);
-                let def = Definition::FunctionDef(node_key.clone());
-                let symbol_id = self.add_or_update_symbol_with_def(&node.name, def.clone());
-                for decorator in &node.decorator_list {
-                    self.visit_decorator(decorator);
-                }
-                let scope_id = self.with_type_params(
-                    &node.name,
-                    &node.type_params,
-                    Some(def.clone()),
-                    Some(symbol_id),
-                    |indexer| {
-                        indexer.visit_parameters(&node.parameters);
-                        for expr in &node.returns {
-                            indexer.visit_annotation(expr);
-                        }
-                        let scope_id = indexer.push_scope(
-                            &node.name,
-                            ScopeKind::Function,
-                            Some(def.clone()),
-                            Some(symbol_id),
-                        );
-                        indexer.visit_body(&node.body);
-                        indexer.pop_scope();
-                        scope_id
-                    },
-                );
-                self.record_scope_for_node(*node_key.erased(), scope_id);
-            }
-            ast::Stmt::Import(ast::StmtImport { names, .. }) => {
-                for alias in names {
-                    let symbol_name = if let Some(asname) = &alias.asname {
-                        asname.id.as_str()
-                    } else {
-                        alias.name.id.split('.').next().unwrap()
-                    };
-
-                    let module = ModuleName::new(&alias.name.id).unwrap();
-
-                    let def = Definition::Import(ImportDefinition {
-                        module: module.clone(),
-                    });
-                    self.add_or_update_symbol_with_def(symbol_name, def);
-                    self.symbol_table_builder
-                        .add_dependency(Dependency::Module(module));
-                }
-            }
-            ast::Stmt::ImportFrom(ast::StmtImportFrom {
-                module,
-                names,
-                level,
-                ..
-            }) => {
-                let module = module.as_ref().and_then(|m| ModuleName::new(&m.id));
-
-                for alias in names {
-                    let symbol_name = if let Some(asname) = &alias.asname {
-                        asname.id.as_str()
-                    } else {
-                        alias.name.id.as_str()
-                    };
-                    let def = Definition::ImportFrom(ImportFromDefinition {
-                        module: module.clone(),
-                        name: alias.name.id.clone(),
-                        level: *level,
-                    });
-                    self.add_or_update_symbol_with_def(symbol_name, def);
-                }
-
-                let dependency = if let Some(module) = module {
-                    match NonZeroU32::new(*level) {
-                        Some(level) => Dependency::Relative {
-                            level,
-                            module: Some(module),
-                        },
-                        None => Dependency::Module(module),
-                    }
-                } else {
-                    Dependency::Relative {
-                        level: NonZeroU32::new(*level)
-                            .expect("Import without a module to have a level > 0"),
-                        module,
-                    }
-                };
-
-                self.symbol_table_builder.add_dependency(dependency);
-            }
-            ast::Stmt::Assign(node) => {
-                debug_assert!(self.current_definition.is_none());
-                self.visit_expr(&node.value);
-                self.current_definition =
-                    Some(Definition::Assignment(TypedNodeKey::from_node(node)));
-                for expr in &node.targets {
-                    self.visit_expr(expr);
-                }
-
-                self.current_definition = None;
-            }
-            ast::Stmt::If(node) => {
-                // TODO detect statically known truthy or falsy test (via type inference, not naive
-                // AST inspection, so we can't simplify here, need to record test expression in CFG
-                // for later checking)
-
-                // we visit the if "test" condition first regardless
-                self.visit_expr(&node.test);
-
-                // create branch node: does the if test pass or not?
-                let if_branch = self.flow_graph_builder.add_branch(self.current_flow_node());
-
-                // visit the body of the `if` clause
-                self.set_current_flow_node(if_branch);
-                self.insert_constraint(&node.test);
-                self.visit_body(&node.body);
-
-                // Flow node for the last if/elif condition branch; represents the "no branch
-                // taken yet" possibility (where "taking a branch" means that the condition in an
-                // if or elif evaluated to true and control flow went into that clause).
-                let mut prior_branch = if_branch;
-
-                // Flow node for the state after the prior if/elif/else clause; represents "we have
-                // taken one of the branches up to this point." Initially set to the post-if-clause
-                // state, later will be set to the phi node joining that possible path with the
-                // possibility that we took a later if/elif/else clause instead.
-                let mut post_prior_clause = self.current_flow_node();
-
-                // Flag to mark if the final clause is an "else" -- if so, that means the "match no
-                // clauses" path is not possible, we have to go through one of the clauses.
-                let mut last_branch_is_else = false;
-
-                for clause in &node.elif_else_clauses {
-                    if let Some(test) = &clause.test {
-                        self.visit_expr(test);
-                        // This is an elif clause. Create a new branch node. Its predecessor is the
-                        // previous branch node, because we can only take one branch in an entire
-                        // if/elif/else chain, so if we take this branch, it can only be because we
-                        // didn't take the previous one.
-                        prior_branch = self.flow_graph_builder.add_branch(prior_branch);
-                        self.set_current_flow_node(prior_branch);
-                        self.insert_constraint(test);
-                    } else {
-                        // This is an else clause. No need to create a branch node; there's no
-                        // branch here, if we haven't taken any previous branch, we definitely go
-                        // into the "else" clause.
-                        self.set_current_flow_node(prior_branch);
-                        last_branch_is_else = true;
-                    }
-                    self.visit_elif_else_clause(clause);
-                    // Update `post_prior_clause` to a new phi node joining the possibility that we
-                    // took any of the previous branches with the possibility that we took the one
-                    // just visited.
-                    post_prior_clause = self
-                        .flow_graph_builder
-                        .add_phi(self.current_flow_node(), post_prior_clause);
-                }
-
-                if !last_branch_is_else {
-                    // Final branch was not an "else", which means it's possible we took zero
-                    // branches in the entire if/elif chain, so we need one more phi node to join
-                    // the "no branches taken" possibility.
-                    post_prior_clause = self
-                        .flow_graph_builder
-                        .add_phi(post_prior_clause, prior_branch);
-                }
-
-                // Onward, with current flow node set to our final Phi node.
-                self.set_current_flow_node(post_prior_clause);
-            }
-            _ => {
-                ast::visitor::source_order::walk_stmt(self, stmt);
-            }
-        }
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct SemanticIndexStorage(KeyValueCache<FileId, Arc<SemanticIndex>>);
-
-impl Deref for SemanticIndexStorage {
-    type Target = KeyValueCache<FileId, Arc<SemanticIndex>>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for SemanticIndexStorage {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::semantic::symbol_table::{Symbol, SymbolIterator};
-    use ruff_python_ast as ast;
-    use ruff_python_ast::ModModule;
-    use ruff_python_parser::{Mode, Parsed};
-
-    use super::{Definition, ScopeKind, SemanticIndex, SymbolId};
-
-    fn parse(code: &str) -> Parsed<ModModule> {
-        ruff_python_parser::parse_unchecked(code, Mode::Module)
-            .try_into_module()
-            .unwrap()
-    }
-
-    fn names<I>(it: SymbolIterator<I>) -> Vec<&str>
-    where
-        I: Iterator<Item = SymbolId>,
-    {
-        let mut symbols: Vec<_> = it.map(Symbol::name).collect();
-        symbols.sort_unstable();
-        symbols
-    }
-
-    #[test]
-    fn empty() {
-        let parsed = parse("");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()).len(), 0);
-    }
-
-    #[test]
-    fn simple() {
-        let parsed = parse("x");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["x"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("x").unwrap())
-                .len(),
-            0
-        );
-    }
-
-    #[test]
-    fn annotation_only() {
-        let parsed = parse("x: int");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["int", "x"]);
-        // TODO record definition
-    }
-
-    #[test]
-    fn import() {
-        let parsed = parse("import foo");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["foo"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("foo").unwrap())
-                .len(),
-            1
-        );
-    }
-
-    #[test]
-    fn import_sub() {
-        let parsed = parse("import foo.bar");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["foo"]);
-    }
-
-    #[test]
-    fn import_as() {
-        let parsed = parse("import foo.bar as baz");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["baz"]);
-    }
-
-    #[test]
-    fn import_from() {
-        let parsed = parse("from bar import foo");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["foo"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("foo").unwrap())
-                .len(),
-            1
-        );
-        assert!(
-            table.root_symbol_id_by_name("foo").is_some_and(|sid| {
-                let s = sid.symbol(&table);
-                s.is_defined() || !s.is_used()
-            }),
-            "symbols that are defined get the defined flag"
-        );
-    }
-
-    #[test]
-    fn assign() {
-        let parsed = parse("x = foo");
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["foo", "x"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("x").unwrap())
-                .len(),
-            1
-        );
-        assert!(
-            table.root_symbol_id_by_name("foo").is_some_and(|sid| {
-                let s = sid.symbol(&table);
-                !s.is_defined() && s.is_used()
-            }),
-            "a symbol used but not defined in a scope should have only the used flag"
-        );
-    }
-
-    #[test]
-    fn class_scope() {
-        let parsed = parse(
-            "
-                class C:
-                    x = 1
-                y = 2
-                ",
-        );
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["C", "y"]);
-        let scopes = table.root_child_scope_ids();
-        assert_eq!(scopes.len(), 1);
-        let c_scope = scopes[0].scope(&table);
-        assert_eq!(c_scope.kind(), ScopeKind::Class);
-        assert_eq!(c_scope.name(), "C");
-        assert_eq!(names(table.symbols_for_scope(scopes[0])), vec!["x"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("C").unwrap())
-                .len(),
-            1
-        );
-    }
-
-    #[test]
-    fn func_scope() {
-        let parsed = parse(
-            "
-                def func():
-                    x = 1
-                y = 2
-                ",
-        );
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["func", "y"]);
-        let scopes = table.root_child_scope_ids();
-        assert_eq!(scopes.len(), 1);
-        let func_scope = scopes[0].scope(&table);
-        assert_eq!(func_scope.kind(), ScopeKind::Function);
-        assert_eq!(func_scope.name(), "func");
-        assert_eq!(names(table.symbols_for_scope(scopes[0])), vec!["x"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("func").unwrap())
-                .len(),
-            1
-        );
-    }
-
-    #[test]
-    fn dupes() {
-        let parsed = parse(
-            "
-                def func():
-                    x = 1
-                def func():
-                    y = 2
-                ",
-        );
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["func"]);
-        let scopes = table.root_child_scope_ids();
-        assert_eq!(scopes.len(), 2);
-        let func_scope_1 = scopes[0].scope(&table);
-        let func_scope_2 = scopes[1].scope(&table);
-        assert_eq!(func_scope_1.kind(), ScopeKind::Function);
-        assert_eq!(func_scope_1.name(), "func");
-        assert_eq!(func_scope_2.kind(), ScopeKind::Function);
-        assert_eq!(func_scope_2.name(), "func");
-        assert_eq!(names(table.symbols_for_scope(scopes[0])), vec!["x"]);
-        assert_eq!(names(table.symbols_for_scope(scopes[1])), vec!["y"]);
-        assert_eq!(
-            table
-                .definitions(table.root_symbol_id_by_name("func").unwrap())
-                .len(),
-            2
-        );
-    }
-
-    #[test]
-    fn generic_func() {
-        let parsed = parse(
-            "
-                def func[T]():
-                    x = 1
-                ",
-        );
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["func"]);
-        let scopes = table.root_child_scope_ids();
-        assert_eq!(scopes.len(), 1);
-        let ann_scope_id = scopes[0];
-        let ann_scope = ann_scope_id.scope(&table);
-        assert_eq!(ann_scope.kind(), ScopeKind::Annotation);
-        assert_eq!(ann_scope.name(), "func");
-        assert_eq!(names(table.symbols_for_scope(ann_scope_id)), vec!["T"]);
-        let scopes = table.child_scope_ids_of(ann_scope_id);
-        assert_eq!(scopes.len(), 1);
-        let func_scope_id = scopes[0];
-        let func_scope = func_scope_id.scope(&table);
-        assert_eq!(func_scope.kind(), ScopeKind::Function);
-        assert_eq!(func_scope.name(), "func");
-        assert_eq!(names(table.symbols_for_scope(func_scope_id)), vec!["x"]);
-    }
-
-    #[test]
-    fn generic_class() {
-        let parsed = parse(
-            "
-                class C[T]:
-                    x = 1
-                ",
-        );
-        let table = SemanticIndex::from_ast(parsed.syntax()).symbol_table;
-        assert_eq!(names(table.root_symbols()), vec!["C"]);
-        let scopes = table.root_child_scope_ids();
-        assert_eq!(scopes.len(), 1);
-        let ann_scope_id = scopes[0];
-        let ann_scope = ann_scope_id.scope(&table);
-        assert_eq!(ann_scope.kind(), ScopeKind::Annotation);
-        assert_eq!(ann_scope.name(), "C");
-        assert_eq!(names(table.symbols_for_scope(ann_scope_id)), vec!["T"]);
-        assert!(
-            table
-                .symbol_by_name(ann_scope_id, "T")
-                .is_some_and(|s| s.is_defined() && !s.is_used()),
-            "type parameters are defined by the scope that introduces them"
-        );
-        let scopes = table.child_scope_ids_of(ann_scope_id);
-        assert_eq!(scopes.len(), 1);
-        let func_scope_id = scopes[0];
-        let func_scope = func_scope_id.scope(&table);
-        assert_eq!(func_scope.kind(), ScopeKind::Class);
-        assert_eq!(func_scope.name(), "C");
-        assert_eq!(names(table.symbols_for_scope(func_scope_id)), vec!["x"]);
-    }
-
-    #[test]
-    fn reachability_trivial() {
-        let parsed = parse("x = 1; x");
-        let ast = parsed.syntax();
-        let index = SemanticIndex::from_ast(ast);
-        let table = &index.symbol_table;
-        let x_sym = table
-            .root_symbol_id_by_name("x")
-            .expect("x symbol should exist");
-        let ast::Stmt::Expr(ast::StmtExpr { value: x_use, .. }) = &ast.body[1] else {
-            panic!("should be an expr")
-        };
-        let x_defs: Vec<_> = index
-            .reachable_definitions(x_sym, x_use)
-            .map(|constrained_definition| constrained_definition.definition)
-            .collect();
-        assert_eq!(x_defs.len(), 1);
-        let Definition::Assignment(node_key) = &x_defs[0] else {
-            panic!("def should be an assignment")
-        };
-        let Some(def_node) = node_key.resolve(ast.into()) else {
-            panic!("node key should resolve")
-        };
-        let ast::Expr::NumberLiteral(ast::ExprNumberLiteral {
-            value: ast::Number::Int(num),
-            ..
-        }) = &*def_node.value
-        else {
-            panic!("should be a number literal")
-        };
-        assert_eq!(*num, 1);
-    }
-
-    #[test]
-    fn expression_scope() {
-        let parsed = parse("x = 1;\ndef test():\n  y = 4");
-        let ast = parsed.syntax();
-        let index = SemanticIndex::from_ast(ast);
-        let table = &index.symbol_table;
-
-        let x_sym = table
-            .root_symbol_by_name("x")
-            .expect("x symbol should exist");
-
-        let x_stmt = ast.body[0].as_assign_stmt().unwrap();
-
-        let x_id = index.expression_id(&x_stmt.targets[0]);
-
-        assert_eq!(table.scope_of_expression(x_id).kind(), ScopeKind::Module);
-        assert_eq!(table.scope_id_of_expression(x_id), x_sym.scope_id());
-
-        let def = ast.body[1].as_function_def_stmt().unwrap();
-        let y_stmt = def.body[0].as_assign_stmt().unwrap();
-        let y_id = index.expression_id(&y_stmt.targets[0]);
-
-        assert_eq!(table.scope_of_expression(y_id).kind(), ScopeKind::Function);
-    }
-}

crates/red_knot/src/semantic/definitions.rs

@@ -1,52 +0,0 @@
-use crate::ast_ids::TypedNodeKey;
-use red_knot_module_resolver::ModuleName;
-use ruff_python_ast as ast;
-use ruff_python_ast::name::Name;
-
-// TODO storing TypedNodeKey for definitions means we have to search to find them again in the AST;
-// this is at best O(log n). If looking up definitions is a bottleneck we should look for
-// alternatives here.
-// TODO intern Definitions in SymbolTable and reference using IDs?
-#[derive(Clone, Debug)]
-pub enum Definition {
-    // For the import cases, we don't need reference to any arbitrary AST subtrees (annotations,
-    // RHS), and referencing just the import statement node is imprecise (a single import statement
-    // can assign many symbols, we'd have to re-search for the one we care about), so we just copy
-    // the small amount of information we need from the AST.
-    Import(ImportDefinition),
-    ImportFrom(ImportFromDefinition),
-    ClassDef(TypedNodeKey<ast::StmtClassDef>),
-    FunctionDef(TypedNodeKey<ast::StmtFunctionDef>),
-    Assignment(TypedNodeKey<ast::StmtAssign>),
-    AnnotatedAssignment(TypedNodeKey<ast::StmtAnnAssign>),
-    NamedExpr(TypedNodeKey<ast::ExprNamed>),
-    /// represents the implicit initial definition of every name as "unbound"
-    Unbound,
-    // TODO with statements, except handlers, function args...
-}
-
-#[derive(Clone, Debug)]
-pub struct ImportDefinition {
-    pub module: ModuleName,
-}
-
-#[derive(Clone, Debug)]
-pub struct ImportFromDefinition {
-    pub module: Option<ModuleName>,
-    pub name: Name,
-    pub level: u32,
-}
-
-impl ImportFromDefinition {
-    pub fn module(&self) -> Option<&ModuleName> {
-        self.module.as_ref()
-    }
-
-    pub fn name(&self) -> &Name {
-        &self.name
-    }
-
-    pub fn level(&self) -> u32 {
-        self.level
-    }
-}

crates/red_knot/src/semantic/flow_graph.rs

@@ -1,270 +0,0 @@
-use super::symbol_table::SymbolId;
-use crate::semantic::{Definition, ExpressionId};
-use ruff_index::{newtype_index, IndexVec};
-use std::iter::FusedIterator;
-use std::ops::Range;
-
-#[newtype_index]
-pub struct FlowNodeId;
-
-#[derive(Debug)]
-pub(crate) enum FlowNode {
-    Start,
-    Definition(DefinitionFlowNode),
-    Branch(BranchFlowNode),
-    Phi(PhiFlowNode),
-    Constraint(ConstraintFlowNode),
-}
-
-/// A point in control flow where a symbol is defined
-#[derive(Debug)]
-pub(crate) struct DefinitionFlowNode {
-    symbol_id: SymbolId,
-    definition: Definition,
-    predecessor: FlowNodeId,
-}
-
-/// A branch in control flow
-#[derive(Debug)]
-pub(crate) struct BranchFlowNode {
-    predecessor: FlowNodeId,
-}
-
-/// A join point where control flow paths come together
-#[derive(Debug)]
-pub(crate) struct PhiFlowNode {
-    first_predecessor: FlowNodeId,
-    second_predecessor: FlowNodeId,
-}
-
-/// A branch test which may apply constraints to a symbol's type
-#[derive(Debug)]
-pub(crate) struct ConstraintFlowNode {
-    predecessor: FlowNodeId,
-    test_expression: ExpressionId,
-}
-
-#[derive(Debug)]
-pub struct FlowGraph {
-    flow_nodes_by_id: IndexVec<FlowNodeId, FlowNode>,
-    expression_map: IndexVec<ExpressionId, FlowNodeId>,
-}
-
-impl FlowGraph {
-    pub fn start() -> FlowNodeId {
-        FlowNodeId::from_usize(0)
-    }
-
-    pub fn for_expr(&self, expr: ExpressionId) -> FlowNodeId {
-        self.expression_map[expr]
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct FlowGraphBuilder {
-    flow_graph: FlowGraph,
-}
-
-impl FlowGraphBuilder {
-    pub(crate) fn new() -> Self {
-        let mut graph = FlowGraph {
-            flow_nodes_by_id: IndexVec::default(),
-            expression_map: IndexVec::default(),
-        };
-        graph.flow_nodes_by_id.push(FlowNode::Start);
-        Self { flow_graph: graph }
-    }
-
-    pub(crate) fn add(&mut self, node: FlowNode) -> FlowNodeId {
-        self.flow_graph.flow_nodes_by_id.push(node)
-    }
-
-    pub(crate) fn add_definition(
-        &mut self,
-        symbol_id: SymbolId,
-        definition: Definition,
-        predecessor: FlowNodeId,
-    ) -> FlowNodeId {
-        self.add(FlowNode::Definition(DefinitionFlowNode {
-            symbol_id,
-            definition,
-            predecessor,
-        }))
-    }
-
-    pub(crate) fn add_branch(&mut self, predecessor: FlowNodeId) -> FlowNodeId {
-        self.add(FlowNode::Branch(BranchFlowNode { predecessor }))
-    }
-
-    pub(crate) fn add_phi(
-        &mut self,
-        first_predecessor: FlowNodeId,
-        second_predecessor: FlowNodeId,
-    ) -> FlowNodeId {
-        self.add(FlowNode::Phi(PhiFlowNode {
-            first_predecessor,
-            second_predecessor,
-        }))
-    }
-
-    pub(crate) fn add_constraint(
-        &mut self,
-        predecessor: FlowNodeId,
-        test_expression: ExpressionId,
-    ) -> FlowNodeId {
-        self.add(FlowNode::Constraint(ConstraintFlowNode {
-            predecessor,
-            test_expression,
-        }))
-    }
-
-    pub(super) fn record_expr(&mut self, node_id: FlowNodeId) -> ExpressionId {
-        self.flow_graph.expression_map.push(node_id)
-    }
-
-    pub(super) fn finish(mut self) -> FlowGraph {
-        self.flow_graph.flow_nodes_by_id.shrink_to_fit();
-        self.flow_graph.expression_map.shrink_to_fit();
-        self.flow_graph
-    }
-}
-
-/// A definition, and the set of constraints between a use and the definition
-#[derive(Debug, Clone)]
-pub struct ConstrainedDefinition {
-    pub definition: Definition,
-    pub constraints: Vec<ExpressionId>,
-}
-
-/// A flow node and the constraints we passed through to reach it
-#[derive(Debug)]
-struct FlowState {
-    node_id: FlowNodeId,
-    constraints_range: Range<usize>,
-}
-
-#[derive(Debug)]
-pub struct ReachableDefinitionsIterator<'a> {
-    flow_graph: &'a FlowGraph,
-    symbol_id: SymbolId,
-    pending: Vec<FlowState>,
-    constraints: Vec<ExpressionId>,
-}
-
-impl<'a> ReachableDefinitionsIterator<'a> {
-    pub fn new(flow_graph: &'a FlowGraph, symbol_id: SymbolId, start_node_id: FlowNodeId) -> Self {
-        Self {
-            flow_graph,
-            symbol_id,
-            pending: vec![FlowState {
-                node_id: start_node_id,
-                constraints_range: 0..0,
-            }],
-            constraints: vec![],
-        }
-    }
-}
-
-impl<'a> Iterator for ReachableDefinitionsIterator<'a> {
-    type Item = ConstrainedDefinition;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let FlowState {
-            mut node_id,
-            mut constraints_range,
-        } = self.pending.pop()?;
-        self.constraints.truncate(constraints_range.end + 1);
-        loop {
-            match &self.flow_graph.flow_nodes_by_id[node_id] {
-                FlowNode::Start => {
-                    // constraints on unbound are irrelevant
-                    return Some(ConstrainedDefinition {
-                        definition: Definition::Unbound,
-                        constraints: vec![],
-                    });
-                }
-                FlowNode::Definition(def_node) => {
-                    if def_node.symbol_id == self.symbol_id {
-                        return Some(ConstrainedDefinition {
-                            definition: def_node.definition.clone(),
-                            constraints: self.constraints[constraints_range].to_vec(),
-                        });
-                    }
-                    node_id = def_node.predecessor;
-                }
-                FlowNode::Branch(branch_node) => {
-                    node_id = branch_node.predecessor;
-                }
-                FlowNode::Phi(phi_node) => {
-                    self.pending.push(FlowState {
-                        node_id: phi_node.first_predecessor,
-                        constraints_range: constraints_range.clone(),
-                    });
-                    node_id = phi_node.second_predecessor;
-                }
-                FlowNode::Constraint(constraint_node) => {
-                    node_id = constraint_node.predecessor;
-                    self.constraints.push(constraint_node.test_expression);
-                    constraints_range.end += 1;
-                }
-            }
-        }
-    }
-}
-
-impl<'a> FusedIterator for ReachableDefinitionsIterator<'a> {}
-
-impl std::fmt::Display for FlowGraph {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        writeln!(f, "flowchart TD")?;
-        for (id, node) in self.flow_nodes_by_id.iter_enumerated() {
-            write!(f, "  id{}", id.as_u32())?;
-            match node {
-                FlowNode::Start => writeln!(f, r"[\Start/]")?,
-                FlowNode::Definition(def_node) => {
-                    writeln!(f, r"(Define symbol {})", def_node.symbol_id.as_u32())?;
-                    writeln!(
-                        f,
-                        r"  id{}-->id{}",
-                        def_node.predecessor.as_u32(),
-                        id.as_u32()
-                    )?;
-                }
-                FlowNode::Branch(branch_node) => {
-                    writeln!(f, r"{{Branch}}")?;
-                    writeln!(
-                        f,
-                        r"  id{}-->id{}",
-                        branch_node.predecessor.as_u32(),
-                        id.as_u32()
-                    )?;
-                }
-                FlowNode::Phi(phi_node) => {
-                    writeln!(f, r"((Phi))")?;
-                    writeln!(
-                        f,
-                        r"  id{}-->id{}",
-                        phi_node.second_predecessor.as_u32(),
-                        id.as_u32()
-                    )?;
-                    writeln!(
-                        f,
-                        r"  id{}-->id{}",
-                        phi_node.first_predecessor.as_u32(),
-                        id.as_u32()
-                    )?;
-                }
-                FlowNode::Constraint(constraint_node) => {
-                    writeln!(f, r"((Constraint))")?;
-                    writeln!(
-                        f,
-                        r"  id{}-->id{}",
-                        constraint_node.predecessor.as_u32(),
-                        id.as_u32()
-                    )?;
-                }
-            }
-        }
-        Ok(())
-    }
-}

crates/red_knot/src/semantic/symbol_table.rs

@@ -1,560 +0,0 @@
-#![allow(dead_code)]
-
-use std::hash::{Hash, Hasher};
-use std::iter::{Copied, DoubleEndedIterator, FusedIterator};
-use std::num::NonZeroU32;
-
-use bitflags::bitflags;
-use hashbrown::hash_map::{Keys, RawEntryMut};
-use red_knot_module_resolver::ModuleName;
-use rustc_hash::{FxHashMap, FxHasher};
-
-use ruff_index::{newtype_index, IndexVec};
-use ruff_python_ast::name::Name;
-
-use crate::ast_ids::NodeKey;
-use crate::semantic::{Definition, ExpressionId};
-
-type Map<K, V> = hashbrown::HashMap<K, V, ()>;
-
-#[newtype_index]
-pub struct ScopeId;
-
-impl ScopeId {
-    pub fn scope(self, table: &SymbolTable) -> &Scope {
-        &table.scopes_by_id[self]
-    }
-}
-
-#[newtype_index]
-pub struct SymbolId;
-
-impl SymbolId {
-    pub fn symbol(self, table: &SymbolTable) -> &Symbol {
-        &table.symbols_by_id[self]
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub enum ScopeKind {
-    Module,
-    Annotation,
-    Class,
-    Function,
-}
-
-#[derive(Debug)]
-pub struct Scope {
-    name: Name,
-    kind: ScopeKind,
-    parent: Option<ScopeId>,
-    children: Vec<ScopeId>,
-    /// the definition (e.g. class or function) that created this scope
-    definition: Option<Definition>,
-    /// the symbol (e.g. class or function) that owns this scope
-    defining_symbol: Option<SymbolId>,
-    /// symbol IDs, hashed by symbol name
-    symbols_by_name: Map<SymbolId, ()>,
-}
-
-impl Scope {
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn kind(&self) -> ScopeKind {
-        self.kind
-    }
-
-    pub fn definition(&self) -> Option<Definition> {
-        self.definition.clone()
-    }
-
-    pub fn defining_symbol(&self) -> Option<SymbolId> {
-        self.defining_symbol
-    }
-}
-
-#[derive(Debug)]
-pub(crate) enum Kind {
-    FreeVar,
-    CellVar,
-    CellVarAssigned,
-    ExplicitGlobal,
-    ImplicitGlobal,
-}
-
-bitflags! {
-    #[derive(Copy,Clone,Debug)]
-    pub struct SymbolFlags: u8 {
-        const IS_USED         = 1 << 0;
-        const IS_DEFINED      = 1 << 1;
-        /// TODO: This flag is not yet set by anything
-        const MARKED_GLOBAL   = 1 << 2;
-        /// TODO: This flag is not yet set by anything
-        const MARKED_NONLOCAL = 1 << 3;
-    }
-}
-
-#[derive(Debug)]
-pub struct Symbol {
-    name: Name,
-    flags: SymbolFlags,
-    scope_id: ScopeId,
-    // kind: Kind,
-}
-
-impl Symbol {
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn scope_id(&self) -> ScopeId {
-        self.scope_id
-    }
-
-    /// Is the symbol used in its containing scope?
-    pub fn is_used(&self) -> bool {
-        self.flags.contains(SymbolFlags::IS_USED)
-    }
-
-    /// Is the symbol defined in its containing scope?
-    pub fn is_defined(&self) -> bool {
-        self.flags.contains(SymbolFlags::IS_DEFINED)
-    }
-
-    // TODO: implement Symbol.kind 2-pass analysis to categorize as: free-var, cell-var,
-    // explicit-global, implicit-global and implement Symbol.kind by modifying the preorder
-    // traversal code
-}
-
-#[derive(Debug, Clone)]
-pub enum Dependency {
-    Module(ModuleName),
-    Relative {
-        level: NonZeroU32,
-        module: Option<ModuleName>,
-    },
-}
-
-/// Table of all symbols in all scopes for a module.
-#[derive(Debug)]
-pub struct SymbolTable {
-    scopes_by_id: IndexVec<ScopeId, Scope>,
-    symbols_by_id: IndexVec<SymbolId, Symbol>,
-    /// the definitions for each symbol
-    defs: FxHashMap<SymbolId, Vec<Definition>>,
-    /// map of AST node (e.g. class/function def) to sub-scope it creates
-    scopes_by_node: FxHashMap<NodeKey, ScopeId>,
-    /// Maps expressions to their enclosing scope.
-    expression_scopes: IndexVec<ExpressionId, ScopeId>,
-    /// dependencies of this module
-    dependencies: Vec<Dependency>,
-}
-
-impl SymbolTable {
-    pub fn dependencies(&self) -> &[Dependency] {
-        &self.dependencies
-    }
-
-    pub const fn root_scope_id() -> ScopeId {
-        ScopeId::from_usize(0)
-    }
-
-    pub fn root_scope(&self) -> &Scope {
-        &self.scopes_by_id[SymbolTable::root_scope_id()]
-    }
-
-    pub fn symbol_ids_for_scope(&self, scope_id: ScopeId) -> Copied<Keys<SymbolId, ()>> {
-        self.scopes_by_id[scope_id].symbols_by_name.keys().copied()
-    }
-
-    pub fn symbols_for_scope(
-        &self,
-        scope_id: ScopeId,
-    ) -> SymbolIterator<Copied<Keys<SymbolId, ()>>> {
-        SymbolIterator {
-            table: self,
-            ids: self.symbol_ids_for_scope(scope_id),
-        }
-    }
-
-    pub fn root_symbol_ids(&self) -> Copied<Keys<SymbolId, ()>> {
-        self.symbol_ids_for_scope(SymbolTable::root_scope_id())
-    }
-
-    pub fn root_symbols(&self) -> SymbolIterator<Copied<Keys<SymbolId, ()>>> {
-        self.symbols_for_scope(SymbolTable::root_scope_id())
-    }
-
-    pub fn child_scope_ids_of(&self, scope_id: ScopeId) -> &[ScopeId] {
-        &self.scopes_by_id[scope_id].children
-    }
-
-    pub fn child_scopes_of(&self, scope_id: ScopeId) -> ScopeIterator<&[ScopeId]> {
-        ScopeIterator {
-            table: self,
-            ids: self.child_scope_ids_of(scope_id),
-        }
-    }
-
-    pub fn root_child_scope_ids(&self) -> &[ScopeId] {
-        self.child_scope_ids_of(SymbolTable::root_scope_id())
-    }
-
-    pub fn root_child_scopes(&self) -> ScopeIterator<&[ScopeId]> {
-        self.child_scopes_of(SymbolTable::root_scope_id())
-    }
-
-    pub fn symbol_id_by_name(&self, scope_id: ScopeId, name: &str) -> Option<SymbolId> {
-        let scope = &self.scopes_by_id[scope_id];
-        let hash = SymbolTable::hash_name(name);
-        let name = Name::new(name);
-        Some(
-            *scope
-                .symbols_by_name
-                .raw_entry()
-                .from_hash(hash, |symid| self.symbols_by_id[*symid].name == name)?
-                .0,
-        )
-    }
-
-    pub fn symbol_by_name(&self, scope_id: ScopeId, name: &str) -> Option<&Symbol> {
-        Some(&self.symbols_by_id[self.symbol_id_by_name(scope_id, name)?])
-    }
-
-    pub fn root_symbol_id_by_name(&self, name: &str) -> Option<SymbolId> {
-        self.symbol_id_by_name(SymbolTable::root_scope_id(), name)
-    }
-
-    pub fn root_symbol_by_name(&self, name: &str) -> Option<&Symbol> {
-        self.symbol_by_name(SymbolTable::root_scope_id(), name)
-    }
-
-    pub fn scope_id_of_symbol(&self, symbol_id: SymbolId) -> ScopeId {
-        self.symbols_by_id[symbol_id].scope_id
-    }
-
-    pub fn scope_of_symbol(&self, symbol_id: SymbolId) -> &Scope {
-        &self.scopes_by_id[self.scope_id_of_symbol(symbol_id)]
-    }
-
-    pub fn scope_id_of_expression(&self, expression: ExpressionId) -> ScopeId {
-        self.expression_scopes[expression]
-    }
-
-    pub fn scope_of_expression(&self, expr_id: ExpressionId) -> &Scope {
-        &self.scopes_by_id[self.scope_id_of_expression(expr_id)]
-    }
-
-    pub fn parent_scopes(
-        &self,
-        scope_id: ScopeId,
-    ) -> ScopeIterator<impl Iterator<Item = ScopeId> + '_> {
-        ScopeIterator {
-            table: self,
-            ids: std::iter::successors(Some(scope_id), |scope| self.scopes_by_id[*scope].parent),
-        }
-    }
-
-    pub fn parent_scope(&self, scope_id: ScopeId) -> Option<ScopeId> {
-        self.scopes_by_id[scope_id].parent
-    }
-
-    pub fn scope_id_for_node(&self, node_key: &NodeKey) -> ScopeId {
-        self.scopes_by_node[node_key]
-    }
-
-    pub fn definitions(&self, symbol_id: SymbolId) -> &[Definition] {
-        self.defs
-            .get(&symbol_id)
-            .map(std::vec::Vec::as_slice)
-            .unwrap_or_default()
-    }
-
-    pub fn all_definitions(&self) -> impl Iterator<Item = (SymbolId, &Definition)> + '_ {
-        self.defs
-            .iter()
-            .flat_map(|(sym_id, defs)| defs.iter().map(move |def| (*sym_id, def)))
-    }
-
-    fn hash_name(name: &str) -> u64 {
-        let mut hasher = FxHasher::default();
-        name.hash(&mut hasher);
-        hasher.finish()
-    }
-}
-
-pub struct SymbolIterator<'a, I> {
-    table: &'a SymbolTable,
-    ids: I,
-}
-
-impl<'a, I> Iterator for SymbolIterator<'a, I>
-where
-    I: Iterator<Item = SymbolId>,
-{
-    type Item = &'a Symbol;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let id = self.ids.next()?;
-        Some(&self.table.symbols_by_id[id])
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.ids.size_hint()
-    }
-}
-
-impl<'a, I> FusedIterator for SymbolIterator<'a, I> where
-    I: Iterator<Item = SymbolId> + FusedIterator
-{
-}
-
-impl<'a, I> DoubleEndedIterator for SymbolIterator<'a, I>
-where
-    I: Iterator<Item = SymbolId> + DoubleEndedIterator,
-{
-    fn next_back(&mut self) -> Option<Self::Item> {
-        let id = self.ids.next_back()?;
-        Some(&self.table.symbols_by_id[id])
-    }
-}
-
-// TODO maybe get rid of this and just do all data access via methods on ScopeId?
-pub struct ScopeIterator<'a, I> {
-    table: &'a SymbolTable,
-    ids: I,
-}
-
-/// iterate (`ScopeId`, `Scope`) pairs for given `ScopeId` iterator
-impl<'a, I> Iterator for ScopeIterator<'a, I>
-where
-    I: Iterator<Item = ScopeId>,
-{
-    type Item = (ScopeId, &'a Scope);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let id = self.ids.next()?;
-        Some((id, &self.table.scopes_by_id[id]))
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.ids.size_hint()
-    }
-}
-
-impl<'a, I> FusedIterator for ScopeIterator<'a, I> where I: Iterator<Item = ScopeId> + FusedIterator {}
-
-impl<'a, I> DoubleEndedIterator for ScopeIterator<'a, I>
-where
-    I: Iterator<Item = ScopeId> + DoubleEndedIterator,
-{
-    fn next_back(&mut self) -> Option<Self::Item> {
-        let id = self.ids.next_back()?;
-        Some((id, &self.table.scopes_by_id[id]))
-    }
-}
-
-#[derive(Debug)]
-pub(super) struct SymbolTableBuilder {
-    symbol_table: SymbolTable,
-}
-
-impl SymbolTableBuilder {
-    pub(super) fn new() -> Self {
-        let mut table = SymbolTable {
-            scopes_by_id: IndexVec::new(),
-            symbols_by_id: IndexVec::new(),
-            defs: FxHashMap::default(),
-            scopes_by_node: FxHashMap::default(),
-            expression_scopes: IndexVec::new(),
-            dependencies: Vec::new(),
-        };
-        table.scopes_by_id.push(Scope {
-            name: Name::new("<module>"),
-            kind: ScopeKind::Module,
-            parent: None,
-            children: Vec::new(),
-            definition: None,
-            defining_symbol: None,
-            symbols_by_name: Map::default(),
-        });
-        Self {
-            symbol_table: table,
-        }
-    }
-
-    pub(super) fn finish(self) -> SymbolTable {
-        let mut symbol_table = self.symbol_table;
-        symbol_table.scopes_by_id.shrink_to_fit();
-        symbol_table.symbols_by_id.shrink_to_fit();
-        symbol_table.defs.shrink_to_fit();
-        symbol_table.scopes_by_node.shrink_to_fit();
-        symbol_table.expression_scopes.shrink_to_fit();
-        symbol_table.dependencies.shrink_to_fit();
-        symbol_table
-    }
-
-    pub(super) fn add_or_update_symbol(
-        &mut self,
-        scope_id: ScopeId,
-        name: &str,
-        flags: SymbolFlags,
-    ) -> SymbolId {
-        let hash = SymbolTable::hash_name(name);
-        let scope = &mut self.symbol_table.scopes_by_id[scope_id];
-        let name = Name::new(name);
-
-        let entry = scope
-            .symbols_by_name
-            .raw_entry_mut()
-            .from_hash(hash, |existing| {
-                self.symbol_table.symbols_by_id[*existing].name == name
-            });
-
-        match entry {
-            RawEntryMut::Occupied(entry) => {
-                if let Some(symbol) = self.symbol_table.symbols_by_id.get_mut(*entry.key()) {
-                    symbol.flags.insert(flags);
-                };
-                *entry.key()
-            }
-            RawEntryMut::Vacant(entry) => {
-                let id = self.symbol_table.symbols_by_id.push(Symbol {
-                    name,
-                    flags,
-                    scope_id,
-                });
-                entry.insert_with_hasher(hash, id, (), |symid| {
-                    SymbolTable::hash_name(&self.symbol_table.symbols_by_id[*symid].name)
-                });
-                id
-            }
-        }
-    }
-
-    pub(super) fn add_definition(&mut self, symbol_id: SymbolId, definition: Definition) {
-        self.symbol_table
-            .defs
-            .entry(symbol_id)
-            .or_default()
-            .push(definition);
-    }
-
-    pub(super) fn add_child_scope(
-        &mut self,
-        parent_scope_id: ScopeId,
-        name: &str,
-        kind: ScopeKind,
-        definition: Option<Definition>,
-        defining_symbol: Option<SymbolId>,
-    ) -> ScopeId {
-        let new_scope_id = self.symbol_table.scopes_by_id.push(Scope {
-            name: Name::new(name),
-            kind,
-            parent: Some(parent_scope_id),
-            children: Vec::new(),
-            definition,
-            defining_symbol,
-            symbols_by_name: Map::default(),
-        });
-        let parent_scope = &mut self.symbol_table.scopes_by_id[parent_scope_id];
-        parent_scope.children.push(new_scope_id);
-        new_scope_id
-    }
-
-    pub(super) fn record_scope_for_node(&mut self, node_key: NodeKey, scope_id: ScopeId) {
-        self.symbol_table.scopes_by_node.insert(node_key, scope_id);
-    }
-
-    pub(super) fn add_dependency(&mut self, dependency: Dependency) {
-        self.symbol_table.dependencies.push(dependency);
-    }
-
-    /// Records the scope for the current expression
-    pub(super) fn record_expression(&mut self, scope: ScopeId) -> ExpressionId {
-        self.symbol_table.expression_scopes.push(scope)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{ScopeKind, SymbolFlags, SymbolTable, SymbolTableBuilder};
-
-    #[test]
-    fn insert_same_name_symbol_twice() {
-        let mut builder = SymbolTableBuilder::new();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let symbol_id_1 =
-            builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::IS_DEFINED);
-        let symbol_id_2 = builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::IS_USED);
-        let table = builder.finish();
-
-        assert_eq!(symbol_id_1, symbol_id_2);
-        assert!(symbol_id_1.symbol(&table).is_used(), "flags must merge");
-        assert!(symbol_id_1.symbol(&table).is_defined(), "flags must merge");
-    }
-
-    #[test]
-    fn insert_different_named_symbols() {
-        let mut builder = SymbolTableBuilder::new();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let symbol_id_1 = builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
-        let symbol_id_2 = builder.add_or_update_symbol(root_scope_id, "bar", SymbolFlags::empty());
-
-        assert_ne!(symbol_id_1, symbol_id_2);
-    }
-
-    #[test]
-    fn add_child_scope_with_symbol() {
-        let mut builder = SymbolTableBuilder::new();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let foo_symbol_top =
-            builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
-        let c_scope = builder.add_child_scope(root_scope_id, "C", ScopeKind::Class, None, None);
-        let foo_symbol_inner = builder.add_or_update_symbol(c_scope, "foo", SymbolFlags::empty());
-
-        assert_ne!(foo_symbol_top, foo_symbol_inner);
-    }
-
-    #[test]
-    fn scope_from_id() {
-        let table = SymbolTableBuilder::new().finish();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let scope = root_scope_id.scope(&table);
-
-        assert_eq!(scope.name.as_str(), "<module>");
-        assert_eq!(scope.kind, ScopeKind::Module);
-    }
-
-    #[test]
-    fn symbol_from_id() {
-        let mut builder = SymbolTableBuilder::new();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let foo_symbol_id =
-            builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
-        let table = builder.finish();
-        let symbol = foo_symbol_id.symbol(&table);
-
-        assert_eq!(symbol.name(), "foo");
-    }
-
-    #[test]
-    fn bigger_symbol_table() {
-        let mut builder = SymbolTableBuilder::new();
-        let root_scope_id = SymbolTable::root_scope_id();
-        let foo_symbol_id =
-            builder.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
-        builder.add_or_update_symbol(root_scope_id, "bar", SymbolFlags::empty());
-        builder.add_or_update_symbol(root_scope_id, "baz", SymbolFlags::empty());
-        builder.add_or_update_symbol(root_scope_id, "qux", SymbolFlags::empty());
-        let table = builder.finish();
-
-        let foo_symbol_id_2 = table
-            .root_symbol_id_by_name("foo")
-            .expect("foo symbol to be found");
-
-        assert_eq!(foo_symbol_id_2, foo_symbol_id);
-    }
-}

crates/red_knot/src/semantic/types/infer.rs

@@ -1,764 +0,0 @@
-#![allow(dead_code)]
-
-use red_knot_module_resolver::ModuleName;
-use ruff_python_ast as ast;
-use ruff_python_ast::AstNode;
-use std::fmt::Debug;
-
-use crate::db::{QueryResult, SemanticDb, SemanticJar};
-
-use crate::module::resolve_module;
-use crate::parse::parse;
-use crate::semantic::types::{ModuleTypeId, Type};
-use crate::semantic::{
-    resolve_global_symbol, semantic_index, ConstrainedDefinition, Definition, GlobalSymbolId,
-    ImportDefinition, ImportFromDefinition,
-};
-use crate::FileId;
-
-// FIXME: Figure out proper dead-lock free synchronisation now that this takes `&db` instead of `&mut db`.
-/// Resolve the public-facing type for a symbol (the type seen by other scopes: other modules, or
-/// nested functions). Because calls to nested functions and imports can occur anywhere in control
-/// flow, this type must be conservative and consider all definitions of the symbol that could
-/// possibly be seen by another scope. Currently we take the most conservative approach, which is
-/// the union of all definitions. We may be able to narrow this in future to eliminate definitions
-/// which can't possibly (or at least likely) be seen by any other scope, so that e.g. we could
-/// infer `Literal["1"]` instead of `Literal[1] | Literal["1"]` for `x` in `x = x; x = str(x);`.
-#[tracing::instrument(level = "trace", skip(db))]
-pub fn infer_symbol_public_type(db: &dyn SemanticDb, symbol: GlobalSymbolId) -> QueryResult<Type> {
-    let index = semantic_index(db, symbol.file_id)?;
-    let defs = index.symbol_table().definitions(symbol.symbol_id).to_vec();
-    let jar: &SemanticJar = db.jar()?;
-
-    if let Some(ty) = jar.type_store.get_cached_symbol_public_type(symbol) {
-        return Ok(ty);
-    }
-
-    let ty = infer_type_from_definitions(db, symbol, defs.iter().cloned())?;
-
-    jar.type_store.cache_symbol_public_type(symbol, ty);
-
-    // TODO record dependencies
-    Ok(ty)
-}
-
-/// Infer type of a symbol as union of the given `Definitions`.
-fn infer_type_from_definitions<T>(
-    db: &dyn SemanticDb,
-    symbol: GlobalSymbolId,
-    definitions: T,
-) -> QueryResult<Type>
-where
-    T: Debug + IntoIterator<Item = Definition>,
-{
-    infer_type_from_constrained_definitions(
-        db,
-        symbol,
-        definitions
-            .into_iter()
-            .map(|definition| ConstrainedDefinition {
-                definition,
-                constraints: vec![],
-            }),
-    )
-}
-
-/// Infer type of a symbol as union of the given `ConstrainedDefinitions`.
-fn infer_type_from_constrained_definitions<T>(
-    db: &dyn SemanticDb,
-    symbol: GlobalSymbolId,
-    constrained_definitions: T,
-) -> QueryResult<Type>
-where
-    T: IntoIterator<Item = ConstrainedDefinition>,
-{
-    let jar: &SemanticJar = db.jar()?;
-    let mut tys = constrained_definitions
-        .into_iter()
-        .map(|def| infer_constrained_definition_type(db, symbol, def.clone()))
-        .peekable();
-    if let Some(first) = tys.next() {
-        if tys.peek().is_some() {
-            Ok(jar.type_store.add_union(
-                symbol.file_id,
-                &Iterator::chain(std::iter::once(first), tys).collect::<QueryResult<Vec<_>>>()?,
-            ))
-        } else {
-            first
-        }
-    } else {
-        Ok(Type::Unknown)
-    }
-}
-
-/// Infer type for a ConstrainedDefinition (intersection of the definition type and the
-/// constraints)
-#[tracing::instrument(level = "trace", skip(db))]
-pub fn infer_constrained_definition_type(
-    db: &dyn SemanticDb,
-    symbol: GlobalSymbolId,
-    constrained_definition: ConstrainedDefinition,
-) -> QueryResult<Type> {
-    let ConstrainedDefinition {
-        definition,
-        constraints,
-    } = constrained_definition;
-    let index = semantic_index(db, symbol.file_id)?;
-    let parsed = parse(db.upcast(), symbol.file_id)?;
-    let mut intersected_types = vec![infer_definition_type(db, symbol, definition)?];
-    for constraint in constraints {
-        if let Some(constraint_type) = infer_constraint_type(
-            db,
-            symbol,
-            index.resolve_expression_id(parsed.syntax(), constraint),
-        )? {
-            intersected_types.push(constraint_type);
-        }
-    }
-    let jar: &SemanticJar = db.jar()?;
-    Ok(jar
-        .type_store
-        .add_intersection(symbol.file_id, &intersected_types, &[]))
-}
-
-/// Infer a type for a Definition
-#[tracing::instrument(level = "trace", skip(db))]
-pub fn infer_definition_type(
-    db: &dyn SemanticDb,
-    symbol: GlobalSymbolId,
-    definition: Definition,
-) -> QueryResult<Type> {
-    let jar: &SemanticJar = db.jar()?;
-    let type_store = &jar.type_store;
-    let file_id = symbol.file_id;
-
-    match definition {
-        Definition::Unbound => Ok(Type::Unbound),
-        Definition::Import(ImportDefinition {
-            module: module_name,
-        }) => {
-            if let Some(module) = resolve_module(db, &module_name)? {
-                Ok(Type::Module(ModuleTypeId { module, file_id }))
-            } else {
-                Ok(Type::Unknown)
-            }
-        }
-        Definition::ImportFrom(ImportFromDefinition {
-            module,
-            name,
-            level,
-        }) => {
-            // TODO relative imports
-            assert!(matches!(level, 0));
-            let module_name =
-                ModuleName::new(module.as_ref().expect("TODO relative imports")).unwrap();
-            let Some(module) = resolve_module(db, &module_name)? else {
-                return Ok(Type::Unknown);
-            };
-
-            if let Some(remote_symbol) = resolve_global_symbol(db, module, &name)? {
-                infer_symbol_public_type(db, remote_symbol)
-            } else {
-                Ok(Type::Unknown)
-            }
-        }
-        Definition::ClassDef(node_key) => {
-            if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
-                Ok(ty)
-            } else {
-                let parsed = parse(db.upcast(), file_id)?;
-                let ast = parsed.syntax();
-                let index = semantic_index(db, file_id)?;
-                let node = node_key.resolve_unwrap(ast.as_any_node_ref());
-
-                let mut bases = Vec::with_capacity(node.bases().len());
-
-                for base in node.bases() {
-                    bases.push(infer_expr_type(db, file_id, base)?);
-                }
-                let scope_id = index.symbol_table().scope_id_for_node(node_key.erased());
-                let ty = type_store.add_class(file_id, &node.name.id, scope_id, bases);
-                type_store.cache_node_type(file_id, *node_key.erased(), ty);
-                Ok(ty)
-            }
-        }
-        Definition::FunctionDef(node_key) => {
-            if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
-                Ok(ty)
-            } else {
-                let parsed = parse(db.upcast(), file_id)?;
-                let ast = parsed.syntax();
-                let index = semantic_index(db, file_id)?;
-                let node = node_key
-                    .resolve(ast.as_any_node_ref())
-                    .expect("node key should resolve");
-
-                let decorator_tys = node
-                    .decorator_list
-                    .iter()
-                    .map(|decorator| infer_expr_type(db, file_id, &decorator.expression))
-                    .collect::<QueryResult<_>>()?;
-                let scope_id = index.symbol_table().scope_id_for_node(node_key.erased());
-                let ty = type_store.add_function(
-                    file_id,
-                    &node.name.id,
-                    symbol.symbol_id,
-                    scope_id,
-                    decorator_tys,
-                );
-                type_store.cache_node_type(file_id, *node_key.erased(), ty);
-                Ok(ty)
-            }
-        }
-        Definition::Assignment(node_key) => {
-            let parsed = parse(db.upcast(), file_id)?;
-            let ast = parsed.syntax();
-            let node = node_key.resolve_unwrap(ast.as_any_node_ref());
-            // TODO handle unpacking assignment
-            infer_expr_type(db, file_id, &node.value)
-        }
-        Definition::AnnotatedAssignment(node_key) => {
-            let parsed = parse(db.upcast(), file_id)?;
-            let ast = parsed.syntax();
-            let node = node_key.resolve_unwrap(ast.as_any_node_ref());
-            // TODO actually look at the annotation
-            let Some(value) = &node.value else {
-                return Ok(Type::Unknown);
-            };
-            // TODO handle unpacking assignment
-            infer_expr_type(db, file_id, value)
-        }
-        Definition::NamedExpr(node_key) => {
-            let parsed = parse(db.upcast(), file_id)?;
-            let ast = parsed.syntax();
-            let node = node_key.resolve_unwrap(ast.as_any_node_ref());
-            infer_expr_type(db, file_id, &node.value)
-        }
-    }
-}
-
-/// Return the type that the given constraint (an expression from a control-flow test) requires the
-/// given symbol to have. For example, returns the Type "~None" as the constraint type if given the
-/// symbol ID for x and the expression ID for `x is not None`. Returns (Rust) None if the given
-/// expression applies no constraints on the given symbol.
-#[tracing::instrument(level = "trace", skip(db))]
-fn infer_constraint_type(
-    db: &dyn SemanticDb,
-    symbol_id: GlobalSymbolId,
-    // TODO this should preferably take an &ast::Expr instead of AnyNodeRef
-    expression: ast::AnyNodeRef,
-) -> QueryResult<Option<Type>> {
-    let file_id = symbol_id.file_id;
-    let index = semantic_index(db, file_id)?;
-    let jar: &SemanticJar = db.jar()?;
-    let symbol_name = symbol_id.symbol_id.symbol(&index.symbol_table).name();
-    // TODO narrowing attributes
-    // TODO narrowing dict keys
-    // TODO isinstance, ==/!=, type(...), literals, bools...
-    match expression {
-        ast::AnyNodeRef::ExprCompare(ast::ExprCompare {
-            left,
-            ops,
-            comparators,
-            ..
-        }) => {
-            // TODO chained comparisons
-            match left.as_ref() {
-                ast::Expr::Name(ast::ExprName { id, .. }) if id == symbol_name => match ops[0] {
-                    ast::CmpOp::Is | ast::CmpOp::IsNot => {
-                        Ok(match infer_expr_type(db, file_id, &comparators[0])? {
-                            Type::None => Some(Type::None),
-                            _ => None,
-                        }
-                        .map(|ty| {
-                            if matches!(ops[0], ast::CmpOp::IsNot) {
-                                jar.type_store.add_intersection(file_id, &[], &[ty])
-                            } else {
-                                ty
-                            }
-                        }))
-                    }
-                    _ => Ok(None),
-                },
-                _ => Ok(None),
-            }
-        }
-        _ => Ok(None),
-    }
-}
-
-/// Infer type of the given expression.
-fn infer_expr_type(db: &dyn SemanticDb, file_id: FileId, expr: &ast::Expr) -> QueryResult<Type> {
-    // TODO cache the resolution of the type on the node
-    let index = semantic_index(db, file_id)?;
-    match expr {
-        ast::Expr::NoneLiteral(_) => Ok(Type::None),
-        ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value, .. }) => {
-            match value {
-                ast::Number::Int(n) => {
-                    // TODO support big int literals
-                    Ok(n.as_i64().map(Type::IntLiteral).unwrap_or(Type::Unknown))
-                }
-                // TODO builtins.float or builtins.complex
-                _ => Ok(Type::Unknown),
-            }
-        }
-        ast::Expr::Name(name) => {
-            // TODO look up in the correct scope, don't assume global
-            if let Some(symbol_id) = index.symbol_table().root_symbol_id_by_name(&name.id) {
-                infer_type_from_constrained_definitions(
-                    db,
-                    GlobalSymbolId { file_id, symbol_id },
-                    index.reachable_definitions(symbol_id, expr),
-                )
-            } else {
-                Ok(Type::Unknown)
-            }
-        }
-        ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => {
-            let value_type = infer_expr_type(db, file_id, value)?;
-            let attr_name = &attr.id;
-            value_type
-                .get_member(db, attr_name)
-                .map(|ty| ty.unwrap_or(Type::Unknown))
-        }
-        ast::Expr::BinOp(ast::ExprBinOp {
-            left, op, right, ..
-        }) => {
-            let left_ty = infer_expr_type(db, file_id, left)?;
-            let right_ty = infer_expr_type(db, file_id, right)?;
-            // TODO add reverse bin op support if right <: left
-            left_ty.resolve_bin_op(db, *op, right_ty)
-        }
-        ast::Expr::Named(ast::ExprNamed { value, .. }) => infer_expr_type(db, file_id, value),
-        ast::Expr::If(ast::ExprIf { body, orelse, .. }) => {
-            // TODO detect statically known truthy or falsy test
-            let body_ty = infer_expr_type(db, file_id, body)?;
-            let else_ty = infer_expr_type(db, file_id, orelse)?;
-            let jar: &SemanticJar = db.jar()?;
-            Ok(jar.type_store.add_union(file_id, &[body_ty, else_ty]))
-        }
-        _ => todo!("expression type resolution for {:?}", expr),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use red_knot_module_resolver::ModuleName;
-    use ruff_python_ast::name::Name;
-    use std::path::PathBuf;
-
-    use crate::db::tests::TestDb;
-    use crate::db::{HasJar, SemanticJar};
-    use crate::module::{resolve_module, set_module_search_paths, ModuleResolutionInputs};
-    use crate::semantic::{infer_symbol_public_type, resolve_global_symbol, Type};
-
-    // TODO with virtual filesystem we shouldn't have to write files to disk for these
-    // tests
-
-    struct TestCase {
-        temp_dir: tempfile::TempDir,
-        db: TestDb,
-
-        src: PathBuf,
-    }
-
-    fn create_test() -> std::io::Result<TestCase> {
-        let temp_dir = tempfile::tempdir()?;
-
-        let src = temp_dir.path().join("src");
-        std::fs::create_dir(&src)?;
-        let src = src.canonicalize()?;
-
-        let search_paths = ModuleResolutionInputs {
-            extra_paths: vec![],
-            workspace_root: src.clone(),
-            site_packages: None,
-            custom_typeshed: None,
-        };
-
-        let mut db = TestDb::default();
-        set_module_search_paths(&mut db, search_paths);
-
-        Ok(TestCase { temp_dir, db, src })
-    }
-
-    fn write_to_path(case: &TestCase, relative_path: &str, contents: &str) -> anyhow::Result<()> {
-        let path = case.src.join(relative_path);
-        std::fs::write(path, contents)?;
-        Ok(())
-    }
-
-    fn get_public_type(
-        case: &TestCase,
-        module_name: &str,
-        variable_name: &str,
-    ) -> anyhow::Result<Type> {
-        let db = &case.db;
-        let module =
-            resolve_module(db, &ModuleName::new(module_name).unwrap())?.expect("Module to exist");
-        let symbol = resolve_global_symbol(db, module, variable_name)?.expect("symbol to exist");
-
-        Ok(infer_symbol_public_type(db, symbol)?)
-    }
-
-    fn assert_public_type(
-        case: &TestCase,
-        module_name: &str,
-        variable_name: &str,
-        type_name: &str,
-    ) -> anyhow::Result<()> {
-        let ty = get_public_type(case, module_name, variable_name)?;
-
-        let jar = HasJar::<SemanticJar>::jar(&case.db)?;
-        assert_eq!(format!("{}", ty.display(&jar.type_store)), type_name);
-        Ok(())
-    }
-
-    #[test]
-    fn follow_import_to_class() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(&case, "a.py", "from b import C as D; E = D")?;
-        write_to_path(&case, "b.py", "class C: pass")?;
-
-        assert_public_type(&case, "a", "E", "Literal[C]")
-    }
-
-    #[test]
-    fn resolve_base_class_by_name() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "mod.py",
-            "
-                class Base: pass
-                class Sub(Base): pass
-            ",
-        )?;
-
-        let ty = get_public_type(&case, "mod", "Sub")?;
-
-        let Type::Class(class_id) = ty else {
-            panic!("Sub is not a Class")
-        };
-        let jar = HasJar::<SemanticJar>::jar(&case.db)?;
-        let base_names: Vec<_> = jar
-            .type_store
-            .get_class(class_id)
-            .bases()
-            .iter()
-            .map(|base_ty| format!("{}", base_ty.display(&jar.type_store)))
-            .collect();
-
-        assert_eq!(base_names, vec!["Literal[Base]"]);
-
-        Ok(())
-    }
-
-    #[test]
-    fn resolve_method() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "mod.py",
-            "
-                class C:
-                    def f(self): pass
-            ",
-        )?;
-
-        let ty = get_public_type(&case, "mod", "C")?;
-
-        let Type::Class(class_id) = ty else {
-            panic!("C is not a Class");
-        };
-
-        let member_ty = class_id
-            .get_own_class_member(&case.db, &Name::new_static("f"))
-            .expect("C.f to resolve");
-
-        let Some(Type::Function(func_id)) = member_ty else {
-            panic!("C.f is not a Function");
-        };
-
-        let jar = HasJar::<SemanticJar>::jar(&case.db)?;
-        let function = jar.type_store.get_function(func_id);
-        assert_eq!(function.name(), "f");
-
-        Ok(())
-    }
-
-    #[test]
-    fn resolve_module_member() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(&case, "a.py", "import b; D = b.C")?;
-        write_to_path(&case, "b.py", "class C: pass")?;
-
-        assert_public_type(&case, "a", "D", "Literal[C]")
-    }
-
-    #[test]
-    fn resolve_literal() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(&case, "a.py", "x = 1")?;
-
-        assert_public_type(&case, "a", "x", "Literal[1]")
-    }
-
-    #[test]
-    fn resolve_union() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                if flag:
-                    x = 1
-                else:
-                    x = 2
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1, 2]")
-    }
-
-    #[test]
-    fn resolve_visible_def() -> anyhow::Result<()> {
-        let case = create_test()?;
-        write_to_path(&case, "a.py", "y = 1; y = 2; x = y")?;
-        assert_public_type(&case, "a", "x", "Literal[2]")
-    }
-
-    #[test]
-    fn join_paths() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                y = 1
-                y = 2
-                if flag:
-                    y = 3
-                x = y
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[2, 3]")
-    }
-
-    #[test]
-    fn maybe_unbound() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                if flag:
-                    y = 1
-                x = y
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1] | Unbound")
-    }
-
-    #[test]
-    fn if_elif_else() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                y = 1
-                y = 2
-                if flag:
-                    y = 3
-                elif flag2:
-                    y = 4
-                else:
-                    r = y
-                    y = 5
-                    s = y
-                x = y
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[3, 4, 5]")?;
-        assert_public_type(&case, "a", "r", "Literal[2]")?;
-        assert_public_type(&case, "a", "s", "Literal[5]")
-    }
-
-    #[test]
-    fn if_elif() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                y = 1
-                y = 2
-                if flag:
-                    y = 3
-                elif flag2:
-                    y = 4
-                x = y
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[2, 3, 4]")
-    }
-
-    #[test]
-    fn literal_int_arithmetic() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                a = 2 + 1
-                b = a - 4
-                c = a * b
-                d = c / 3
-                e = 5 % 3
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "a", "Literal[3]")?;
-        assert_public_type(&case, "a", "b", "Literal[-1]")?;
-        assert_public_type(&case, "a", "c", "Literal[-3]")?;
-        assert_public_type(&case, "a", "d", "Literal[-1]")?;
-        assert_public_type(&case, "a", "e", "Literal[2]")
-    }
-
-    #[test]
-    fn walrus() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                x = (y := 1) + 1
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[2]")?;
-        assert_public_type(&case, "a", "y", "Literal[1]")
-    }
-
-    #[test]
-    fn ifexpr() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                x = 1 if flag else 2
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1, 2]")
-    }
-
-    #[test]
-    fn ifexpr_walrus() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                y = z = 0
-                x = (y := 1) if flag else (z := 2)
-                a = y
-                b = z
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1, 2]")?;
-        assert_public_type(&case, "a", "a", "Literal[0, 1]")?;
-        assert_public_type(&case, "a", "b", "Literal[0, 2]")
-    }
-
-    #[test]
-    fn ifexpr_walrus_2() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                y = 0
-                (y := 1) if flag else (y := 2)
-                a = y
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "a", "Literal[1, 2]")
-    }
-
-    #[test]
-    fn ifexpr_nested() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                x = 1 if flag else 2 if flag2 else 3
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1, 2, 3]")
-    }
-
-    #[test]
-    fn none() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                x = 1 if flag else None
-            ",
-        )?;
-
-        assert_public_type(&case, "a", "x", "Literal[1] | None")
-    }
-
-    #[test]
-    fn narrow_none() -> anyhow::Result<()> {
-        let case = create_test()?;
-
-        write_to_path(
-            &case,
-            "a.py",
-            "
-                x = 1 if flag else None
-                y = 0
-                if x is not None:
-                    y = x
-                z = y
-            ",
-        )?;
-
-        // TODO normalization of unions and intersections: this type is technically correct but
-        // begging for normalization
-        assert_public_type(&case, "a", "z", "Literal[0] | Literal[1] | None & ~None")
-    }
-}

crates/red_knot/src/source.rs

@@ -1,105 +0,0 @@
-use std::ops::{Deref, DerefMut};
-use std::sync::Arc;
-
-use ruff_notebook::Notebook;
-use ruff_python_ast::PySourceType;
-
-use crate::cache::KeyValueCache;
-use crate::db::{QueryResult, SourceDb};
-use crate::files::FileId;
-
-#[tracing::instrument(level = "debug", skip(db))]
-pub(crate) fn source_text(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Source> {
-    let jar = db.jar()?;
-    let sources = &jar.sources;
-
-    sources.get(&file_id, |file_id| {
-        let path = db.file_path(*file_id);
-
-        let source_text = std::fs::read_to_string(&path).unwrap_or_else(|err| {
-            tracing::error!("Failed to read file '{path:?}: {err}'. Falling back to empty text");
-            String::new()
-        });
-
-        let python_ty = PySourceType::from(&path);
-
-        let kind = match python_ty {
-            PySourceType::Python => {
-                SourceKind::Python(Arc::from(source_text))
-            }
-            PySourceType::Stub => SourceKind::Stub(Arc::from(source_text)),
-            PySourceType::Ipynb => {
-                let notebook = Notebook::from_source_code(&source_text).unwrap_or_else(|err| {
-                    // TODO should this be changed to never fail?
-                    // or should we instead add a diagnostic somewhere? But what would we return in this case?
-                    tracing::error!(
-                        "Failed to parse notebook '{path:?}: {err}'. Falling back to an empty notebook"
-                    );
-                    Notebook::from_source_code("").unwrap()
-                });
-
-                SourceKind::IpyNotebook(Arc::new(notebook))
-            }
-        };
-
-        Ok(Source { kind })
-    })
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum SourceKind {
-    Python(Arc<str>),
-    Stub(Arc<str>),
-    IpyNotebook(Arc<Notebook>),
-}
-
-impl<'a> From<&'a SourceKind> for PySourceType {
-    fn from(value: &'a SourceKind) -> Self {
-        match value {
-            SourceKind::Python(_) => PySourceType::Python,
-            SourceKind::Stub(_) => PySourceType::Stub,
-            SourceKind::IpyNotebook(_) => PySourceType::Ipynb,
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Source {
-    kind: SourceKind,
-}
-
-impl Source {
-    pub fn python<T: Into<Arc<str>>>(source: T) -> Self {
-        Self {
-            kind: SourceKind::Python(source.into()),
-        }
-    }
-    pub fn kind(&self) -> &SourceKind {
-        &self.kind
-    }
-
-    pub fn text(&self) -> &str {
-        match &self.kind {
-            SourceKind::Python(text) => text,
-            SourceKind::Stub(text) => text,
-            SourceKind::IpyNotebook(notebook) => notebook.source_code(),
-        }
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct SourceStorage(pub(crate) KeyValueCache<FileId, Source>);
-
-impl Deref for SourceStorage {
-    type Target = KeyValueCache<FileId, Source>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for SourceStorage {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}

crates/red_knot/src/watch.rs

@@ -1,10 +1,10 @@
 use std::path::Path;
 
+use crate::program::{FileChangeKind, FileWatcherChange};
 use anyhow::Context;
 use notify::event::{CreateKind, RemoveKind};
 use notify::{recommended_watcher, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};
-
+use ruff_db::file_system::FileSystemPath;
-use crate::program::{FileChangeKind, FileWatcherChange};
 
 pub struct FileWatcher {
     watcher: RecommendedWatcher,
@@ -50,7 +50,12 @@ impl FileWatcher {
 
                     for path in event.paths {
                         if path.is_file() {
-                            changes.push(FileWatcherChange::new(path, change_kind));
+                            if let Some(fs_path) = FileSystemPath::from_std_path(&path) {
+                                changes.push(FileWatcherChange::new(
+                                    fs_path.to_path_buf(),
+                                    change_kind,
+                                ));
+                            }
                         }
                     }
 

crates/red_knot_module_resolver/src/resolver.rs

@@ -55,8 +55,8 @@ pub(crate) fn resolve_module_query<'db>(
 /// Resolves the module for the given path.
 ///
 /// Returns `None` if the path is not a module locatable via `sys.path`.
-#[tracing::instrument(level = "debug", skip(db))]
+#[allow(unused)]
-pub fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option<Module> {
+pub(crate) fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option<Module> {
     // It's not entirely clear on first sight why this method calls `file_to_module` instead of
     // it being the other way round, considering that the first thing that `file_to_module` does
     // is to retrieve the file's path.
@@ -73,7 +73,6 @@ pub fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option<Module> {
 ///
 /// Returns `None` if the file is not a module locatable via `sys.path`.
 #[salsa::tracked]
-#[allow(unused)]
 pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option<Module> {
     let _span = tracing::trace_span!("file_to_module", ?file).entered();
 
@@ -367,7 +366,6 @@ impl PackageKind {
 
 #[cfg(test)]
 mod tests {
-
     use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};
     use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath};
 

crates/red_knot_python_semantic/src/semantic_index.rs

@@ -59,7 +59,7 @@ pub(crate) fn root_scope(db: &dyn Db, file: VfsFile) -> ScopeId<'_> {
 
 /// Returns the symbol with the given name in `file`'s public scope or `None` if
 /// no symbol with the given name exists.
-pub fn public_symbol<'db>(
+pub(crate) fn public_symbol<'db>(
     db: &'db dyn Db,
     file: VfsFile,
     name: &str,
@@ -72,7 +72,7 @@ pub fn public_symbol<'db>(
 
 /// The symbol tables for an entire file.
 #[derive(Debug)]
-pub struct SemanticIndex<'db> {
+pub(crate) struct SemanticIndex<'db> {
     /// List of all symbol tables in this file, indexed by scope.
     symbol_tables: IndexVec<FileScopeId, Arc<SymbolTable<'db>>>,
 

crates/red_knot_python_semantic/src/types.rs

@@ -47,8 +47,9 @@ pub(crate) fn public_symbol_ty<'db>(db: &'db dyn Db, symbol: PublicSymbolId<'db>
     inference.symbol_ty(symbol.scoped_symbol_id(db))
 }
 
-/// Shorthand for `public_symbol_ty` that takes a symbol name instead of a [`PublicSymbolId`].
+/// Shorthand for [`public_symbol_ty()`] that takes a symbol name instead of a [`PublicSymbolId`].
-pub fn public_symbol_ty_by_name<'db>(
+#[allow(unused)]
+pub(crate) fn public_symbol_ty_by_name<'db>(
     db: &'db dyn Db,
     file: VfsFile,
     name: &str,

crates/red_knot_python_semantic/src/types/infer.rs

@@ -1,4 +1,5 @@
 use rustc_hash::FxHashMap;
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use red_knot_module_resolver::resolve_module;
@@ -487,21 +488,21 @@ impl<'db> TypeInferenceBuilder<'db> {
 
         match ctx {
             ExprContext::Load => {
-                if let Some(symbol_id) = self
+                let ancestors = self.index.ancestor_scopes(self.file_scope_id);
-                    .index
-                    .symbol_table(self.file_scope_id)
-                    .symbol_id_by_name(id)
-                {
-                    self.local_definition_ty(symbol_id)
-                } else {
-                    let ancestors = self.index.ancestor_scopes(self.file_scope_id).skip(1);
 
                 for (ancestor_id, _) in ancestors {
                     // TODO: Skip over class scopes unless the they are a immediately-nested type param scope.
                     // TODO: Support built-ins
 
+                    let (symbol_table, ancestor_scope) = if ancestor_id == self.file_scope_id {
+                        (Cow::Borrowed(&self.symbol_table), None)
+                    } else {
                         let ancestor_scope = ancestor_id.to_scope_id(self.db, self.file_id);
-                        let symbol_table = symbol_table(self.db, ancestor_scope);
+                        (
+                            Cow::Owned(symbol_table(self.db, ancestor_scope)),
+                            Some(ancestor_scope),
+                        )
+                    };
 
                     if let Some(symbol_id) = symbol_table.symbol_id_by_name(id) {
                         let symbol = symbol_table.symbol(symbol_id);
@@ -510,13 +511,16 @@ impl<'db> TypeInferenceBuilder<'db> {
                             continue;
                         }
 
+                        return if let Some(ancestor_scope) = ancestor_scope {
                             let types = infer_types(self.db, ancestor_scope);
-                            return types.symbol_ty(symbol_id);
+                            types.symbol_ty(symbol_id)
+                        } else {
+                            self.local_definition_ty(symbol_id)
+                        };
                     }
                 }
                 Type::Unknown
             }
-            }
             ExprContext::Del => Type::None,
             ExprContext::Invalid => Type::Unknown,
             ExprContext::Store => Type::None,

crates/ruff_db/src/file_system/os.rs

@@ -2,6 +2,7 @@ use filetime::FileTime;
 
 use crate::file_system::{FileSystem, FileSystemPath, FileType, Metadata, Result};
 
+#[derive(Default)]
 pub struct OsFileSystem;
 
 impl OsFileSystem {

crates/ruff_db/src/source.rs

@@ -1,9 +1,9 @@
+use countme::Count;
+use ruff_source_file::LineIndex;
 use salsa::DebugWithDb;
 use std::ops::Deref;
 use std::sync::Arc;
 
-use ruff_source_file::LineIndex;
-
 use crate::vfs::VfsFile;
 use crate::Db;
 
@@ -16,6 +16,7 @@ pub fn source_text(db: &dyn Db, file: VfsFile) -> SourceText {
 
     SourceText {
         inner: Arc::from(content),
+        count: Count::new(),
     }
 }
 
@@ -35,6 +36,7 @@ pub fn line_index(db: &dyn Db, file: VfsFile) -> LineIndex {
 #[derive(Clone, Eq, PartialEq)]
 pub struct SourceText {
     inner: Arc<str>,
+    count: Count<Self>,
 }
 
 impl SourceText {

crates/ruff_db/src/vfs.rs

@@ -104,6 +104,7 @@ impl Vfs {
     ///
     /// The operation always succeeds even if the path doesn't exist on disk, isn't accessible or if the path points to a directory.
     /// In these cases, a file with status [`FileStatus::Deleted`] is returned.
+    #[tracing::instrument(level = "debug", skip(self, db))]
     fn file_system(&self, db: &dyn Db, path: &FileSystemPath) -> VfsFile {
         *self
             .inner
@@ -135,6 +136,7 @@ impl Vfs {
 
     /// Looks up a vendored file by its path. Returns `Some` if a vendored file for the given path
     /// exists and `None` otherwise.
+    #[tracing::instrument(level = "debug", skip(self, db))]
     fn vendored(&self, db: &dyn Db, path: &VendoredPath) -> Option<VfsFile> {
         let file = match self
             .inner