[red-knot] Extract red_knot_python_semantic crate (#11926)

Cargo.lock

@@ -1978,6 +1978,7 @@ dependencies = [
  "notify",
  "parking_lot",
  "rayon",
+ "red_knot_python_semantic",
  "ruff_index",
  "ruff_notebook",
  "ruff_python_ast",
@@ -1994,6 +1995,28 @@ dependencies = [
  "zip",
 ]
 
+[[package]]
+name = "red_knot_python_semantic"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "bitflags 2.5.0",
+ "hashbrown 0.14.5",
+ "indexmap",
+ "ruff_db",
+ "ruff_index",
+ "ruff_python_ast",
+ "ruff_python_parser",
+ "ruff_python_stdlib",
+ "ruff_text_size",
+ "rustc-hash",
+ "salsa-2022",
+ "smallvec",
+ "smol_str",
+ "tempfile",
+ "tracing",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.4.1"
@@ -2471,12 +2494,8 @@ dependencies = [
 name = "ruff_python_semantic"
 version = "0.0.0"
 dependencies = [
- "anyhow",
  "bitflags 2.5.0",
- "hashbrown 0.14.5",
- "indexmap",
  "is-macro",
- "ruff_db",
  "ruff_index",
  "ruff_python_ast",
  "ruff_python_parser",
@@ -2484,11 +2503,6 @@ dependencies = [
  "ruff_source_file",
  "ruff_text_size",
  "rustc-hash",
- "salsa-2022",
- "smallvec",
- "smol_str",
- "tempfile",
- "tracing",
 ]
 
 [[package]]

Cargo.toml

@@ -35,6 +35,8 @@ ruff_source_file = { path = "crates/ruff_source_file" }
 ruff_text_size = { path = "crates/ruff_text_size" }
 ruff_workspace = { path = "crates/ruff_workspace" }
 
+red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
+
 aho-corasick = { version = "1.1.3" }
 annotate-snippets = { version = "0.9.2", features = ["color"] }
 anyhow = { version = "1.0.80" }

crates/red_knot/Cargo.toml

@@ -12,6 +12,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+red_knot_python_semantic = { workspace = true }
+
 ruff_python_parser = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_stdlib = { workspace = true }

crates/red_knot/src/module.rs

@@ -7,6 +7,8 @@ use std::sync::Arc;
 use dashmap::mapref::entry::Entry;
 use smol_str::SmolStr;
 
+use red_knot_python_semantic::module::ModuleKind;
+
 use crate::db::{QueryResult, SemanticDb, SemanticJar};
 use crate::files::FileId;
 use crate::semantic::Dependency;
@@ -177,15 +179,6 @@ impl std::fmt::Display for ModuleName {
     }
 }
 
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
-pub enum ModuleKind {
-    /// A single-file module (e.g. `foo.py` or `foo.pyi`)
-    Module,
-
-    /// A python package (`foo/__init__.py` or `foo/__init__.pyi`)
-    Package,
-}
-
 /// A search path in which to search modules.
 /// Corresponds to a path in [`sys.path`](https://docs.python.org/3/library/sys_path_init.html) at runtime.
 ///

crates/red_knot_python_semantic/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "red_knot_python_semantic"
+version = "0.0.0"
+publish = false
+authors = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+homepage = { workspace = true }
+documentation = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+ruff_db = { workspace = true }
+ruff_index = { workspace = true }
+ruff_python_ast = { workspace = true }
+ruff_python_stdlib = { workspace = true }
+ruff_text_size = { workspace = true }
+
+bitflags = { workspace = true }
+indexmap = { workspace = true }
+salsa = { workspace = true }
+smallvec = { workspace = true }
+smol_str = { workspace = true }
+tracing = { workspace = true }
+rustc-hash = { workspace = true }
+hashbrown = { workspace = true }
+
+[dev-dependencies]
+anyhow = { workspace = true }
+ruff_python_parser = { workspace = true }
+tempfile = { workspace = true }
+
+[lints]
+workspace = true
+

crates/red_knot_python_semantic/src/ast_node_ref.rs

@@ -93,7 +93,7 @@ unsafe impl<T> Sync for AstNodeRef<T> where T: Sync {}
 
 #[cfg(test)]
 mod tests {
-    use crate::red_knot::ast_node_ref::AstNodeRef;
+    use crate::ast_node_ref::AstNodeRef;
     use ruff_db::parsed::ParsedModule;
     use ruff_python_ast::PySourceType;
     use ruff_python_parser::parse_unchecked_source;

crates/red_knot_python_semantic/src/db.rs

@@ -6,11 +6,10 @@ use crate::module::resolver::{
     file_to_module, internal::ModuleNameIngredient, internal::ModuleResolverSearchPaths,
     resolve_module_query,
 };
-use crate::red_knot::semantic_index::symbol::{
+
-    public_symbols_map, scopes_map, PublicSymbolId, ScopeId,
+use crate::semantic_index::symbol::{public_symbols_map, scopes_map, PublicSymbolId, ScopeId};
-};
+use crate::semantic_index::{root_scope, semantic_index, symbol_table};
-use crate::red_knot::semantic_index::{root_scope, semantic_index, symbol_table};
+use crate::types::{infer_types, public_symbol_ty};
-use crate::red_knot::types::{infer_types, public_symbol_ty};
 
 #[salsa::jar(db=Db)]
 pub struct Jar(

crates/red_knot_python_semantic/src/lib.rs

@@ -0,0 +1,13 @@
+pub mod ast_node_ref;
+mod db;
+pub mod module;
+pub mod name;
+mod node_key;
+pub mod semantic_index;
+pub mod types;
+
+type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;
+
+pub use db::{Db, Jar};
+use rustc_hash::FxHasher;
+use std::hash::BuildHasherDefault;

crates/red_knot_python_semantic/src/mod.rs

@@ -1,8 +1,10 @@
-use rustc_hash::FxHasher;
 use std::hash::BuildHasherDefault;
 
+use rustc_hash::FxHasher;
+
 pub mod ast_node_ref;
 mod node_key;
 pub mod semantic_index;
 pub mod types;
+
 pub(crate) type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;

crates/red_knot_python_semantic/src/module.rs

@@ -46,7 +46,7 @@ impl ModuleName {
     /// ## Examples
     ///
     /// ```
-    /// use ruff_python_semantic::module::ModuleName;
+    /// use red_knot_python_semantic::module::ModuleName;
     ///
     /// assert_eq!(ModuleName::new_static("foo.bar").as_deref(), Some("foo.bar"));
     /// assert_eq!(ModuleName::new_static(""), None);
@@ -78,7 +78,7 @@ impl ModuleName {
     /// # Examples
     ///
     /// ```
-    /// use ruff_python_semantic::module::ModuleName;
+    /// use red_knot_python_semantic::module::ModuleName;
     ///
     /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::<Vec<_>>(), vec!["foo", "bar", "baz"]);
     /// ```
@@ -91,7 +91,7 @@ impl ModuleName {
     /// # Examples
     ///
     /// ```
-    /// use ruff_python_semantic::module::ModuleName;
+    /// use red_knot_python_semantic::module::ModuleName;
     ///
     /// assert_eq!(ModuleName::new_static("foo.bar").unwrap().parent(), Some(ModuleName::new_static("foo").unwrap()));
     /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap()));
@@ -110,7 +110,7 @@ impl ModuleName {
     /// # Examples
     ///
     /// ```
-    /// use ruff_python_semantic::module::ModuleName;
+    /// use red_knot_python_semantic::module::ModuleName;
     ///
     /// assert!(ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap()));
     ///
@@ -312,7 +312,7 @@ struct ModuleSearchPathInner {
 /// for the standard library are moved higher up to match Python's semantics at runtime.
 ///
 /// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, is_macro::Is)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub enum ModuleSearchPathKind {
     /// "Extra" paths provided by the user in a config file, env var or CLI flag.
     /// E.g. mypy's `MYPYPATH` env var, or pyright's `stubPath` configuration setting

crates/red_knot_python_semantic/src/semantic_index.rs

@@ -8,10 +8,10 @@ use ruff_db::vfs::VfsFile;
 use ruff_index::{IndexSlice, IndexVec};
 use ruff_python_ast as ast;
 
-use crate::red_knot::node_key::NodeKey;
+use crate::node_key::NodeKey;
-use crate::red_knot::semantic_index::ast_ids::{AstId, AstIds, ScopeClassId, ScopeFunctionId};
+use crate::semantic_index::ast_ids::{AstId, AstIds, ScopeClassId, ScopeFunctionId};
-use crate::red_knot::semantic_index::builder::SemanticIndexBuilder;
+use crate::semantic_index::builder::SemanticIndexBuilder;
-use crate::red_knot::semantic_index::symbol::{
+use crate::semantic_index::symbol::{
     FileScopeId, PublicSymbolId, Scope, ScopeId, ScopeKind, ScopedSymbolId, SymbolTable,
 };
 use crate::Db;
@@ -272,8 +272,8 @@ mod tests {
     use ruff_db::vfs::{system_path_to_file, VfsFile};
 
     use crate::db::tests::TestDb;
-    use crate::red_knot::semantic_index::symbol::{FileScopeId, ScopeKind, SymbolTable};
+    use crate::semantic_index::symbol::{FileScopeId, ScopeKind, SymbolTable};
-    use crate::red_knot::semantic_index::{root_scope, semantic_index, symbol_table};
+    use crate::semantic_index::{root_scope, semantic_index, symbol_table};
 
     struct TestCase {
         db: TestDb,

crates/red_knot_python_semantic/src/semantic_index/ast_ids.rs

@@ -6,10 +6,10 @@ use ruff_index::{newtype_index, IndexVec};
 use ruff_python_ast as ast;
 use ruff_python_ast::AnyNodeRef;
 
-use crate::red_knot::ast_node_ref::AstNodeRef;
+use crate::ast_node_ref::AstNodeRef;
-use crate::red_knot::node_key::NodeKey;
+use crate::node_key::NodeKey;
-use crate::red_knot::semantic_index::semantic_index;
+use crate::semantic_index::semantic_index;
-use crate::red_knot::semantic_index::symbol::{FileScopeId, ScopeId};
+use crate::semantic_index::symbol::{FileScopeId, ScopeId};
 use crate::Db;
 
 /// AST ids for a single scope.
@@ -98,7 +98,6 @@ pub trait AstIdNode {
     /// ## Panics
     /// May panic if the node does not belongs to `file`'s AST or is outside of `scope`. It may also
     /// return an incorrect node if that's the case.
-
     fn ast_id(&self, db: &dyn Db, file: VfsFile, scope: FileScopeId) -> AstId<Self::ScopeId>;
 
     /// Resolves the AST node for `id`.

crates/red_knot_python_semantic/src/semantic_index/builder.rs

@@ -8,18 +8,16 @@ use ruff_python_ast as ast;
 use ruff_python_ast::visitor::{walk_expr, walk_stmt, Visitor};
 
 use crate::name::Name;
-use crate::red_knot::node_key::NodeKey;
+use crate::node_key::NodeKey;
-use crate::red_knot::semantic_index::ast_ids::{
+use crate::semantic_index::ast_ids::{
     AstId, AstIdsBuilder, ScopeAssignmentId, ScopeClassId, ScopeFunctionId, ScopeImportFromId,
     ScopeImportId, ScopeNamedExprId,
 };
-use crate::red_knot::semantic_index::definition::{
+use crate::semantic_index::definition::{Definition, ImportDefinition, ImportFromDefinition};
-    Definition, ImportDefinition, ImportFromDefinition,
+use crate::semantic_index::symbol::{
-};
-use crate::red_knot::semantic_index::symbol::{
     FileScopeId, FileSymbolId, Scope, ScopedSymbolId, SymbolFlags, SymbolTableBuilder,
 };
-use crate::red_knot::semantic_index::{NodeWithScopeId, SemanticIndex};
+use crate::semantic_index::{NodeWithScopeId, SemanticIndex};
 
 pub(super) struct SemanticIndexBuilder<'a> {
     // Builder state

crates/red_knot_python_semantic/src/semantic_index/definition.rs

@@ -1,4 +1,4 @@
-use crate::red_knot::semantic_index::ast_ids::{
+use crate::semantic_index::ast_ids::{
     ScopeAnnotatedAssignmentId, ScopeAssignmentId, ScopeClassId, ScopeFunctionId,
     ScopeImportFromId, ScopeImportId, ScopeNamedExprId,
 };

crates/red_knot_python_semantic/src/semantic_index/symbol.rs

@@ -14,8 +14,8 @@ use ruff_db::vfs::VfsFile;
 use ruff_index::{newtype_index, IndexVec};
 
 use crate::name::Name;
-use crate::red_knot::semantic_index::definition::Definition;
+use crate::semantic_index::definition::Definition;
-use crate::red_knot::semantic_index::{root_scope, semantic_index, symbol_table, SymbolMap};
+use crate::semantic_index::{root_scope, semantic_index, symbol_table, SymbolMap};
 use crate::Db;
 
 #[derive(Eq, PartialEq, Debug)]

crates/red_knot_python_semantic/src/types.rs

@@ -6,14 +6,14 @@ use ruff_index::newtype_index;
 use ruff_python_ast as ast;
 
 use crate::name::Name;
-use crate::red_knot::semantic_index::ast_ids::{AstIdNode, ScopeAstIdNode};
+use crate::semantic_index::ast_ids::{AstIdNode, ScopeAstIdNode};
-use crate::red_knot::semantic_index::symbol::{FileScopeId, PublicSymbolId, ScopeId};
+use crate::semantic_index::symbol::{FileScopeId, PublicSymbolId, ScopeId};
-use crate::red_knot::semantic_index::{
+use crate::semantic_index::{
     public_symbol, root_scope, semantic_index, symbol_table, NodeWithScopeId,
 };
-use crate::red_knot::types::infer::{TypeInference, TypeInferenceBuilder};
+use crate::types::infer::{TypeInference, TypeInferenceBuilder};
-use crate::red_knot::FxIndexSet;
 use crate::Db;
+use crate::FxIndexSet;
 
 mod display;
 mod infer;
@@ -62,7 +62,7 @@ pub(crate) fn expression_ty(db: &dyn Db, file: VfsFile, expression: &ast::Expr)
 /// This being a query ensures that the invalidation short-circuits if the type of this symbol didn't change.
 #[salsa::tracked]
 pub(crate) fn public_symbol_ty(db: &dyn Db, symbol: PublicSymbolId) -> Type {
-    let _ = tracing::debug_span!("public_symbol_ty", "{:?}", symbol.debug(db));
+    let _ = tracing::debug_span!("public_symbol_ty", symbol = ?symbol.debug(db)).enter();
 
     let file = symbol.file(db);
     let scope = root_scope(db, file);
@@ -71,7 +71,7 @@ pub(crate) fn public_symbol_ty(db: &dyn Db, symbol: PublicSymbolId) -> Type {
     inference.symbol_ty(symbol.scoped_symbol_id(db))
 }
 
-/// Shorthand for [`public_symbol_ty()`] that takes a symbol name instead of a [`PublicSymbolId`].
+/// Shorthand for `public_symbol_ty` that takes a symbol name instead of a [`PublicSymbolId`].
 pub fn public_symbol_ty_by_name(db: &dyn Db, file: VfsFile, name: &str) -> Option<Type> {
     let symbol = public_symbol(db, file, name)?;
     Some(public_symbol_ty(db, symbol))
@@ -500,10 +500,8 @@ mod tests {
         assert_will_not_run_function_query, assert_will_run_function_query, TestDb,
     };
     use crate::module::resolver::{set_module_resolution_settings, ModuleResolutionSettings};
-    use crate::red_knot::semantic_index::root_scope;
+    use crate::semantic_index::root_scope;
-    use crate::red_knot::types::{
+    use crate::types::{expression_ty, infer_types, public_symbol_ty_by_name, TypingContext};
-        expression_ty, infer_types, public_symbol_ty_by_name, TypingContext,
-    };
 
     fn setup_db() -> TestDb {
         let mut db = TestDb::new();

crates/red_knot_python_semantic/src/types/display.rs

@@ -2,7 +2,7 @@
 
 use std::fmt::{Display, Formatter};
 
-use crate::red_knot::types::{IntersectionType, Type, TypingContext, UnionType};
+use crate::types::{IntersectionType, Type, TypingContext, UnionType};
 
 impl Type {
     pub fn display<'a>(&'a self, context: &'a TypingContext) -> DisplayType<'a> {

crates/red_knot_python_semantic/src/types/infer.rs

@@ -10,15 +10,11 @@ use ruff_python_ast::{ExprContext, TypeParams};
 use crate::module::resolver::resolve_module;
 use crate::module::ModuleName;
 use crate::name::Name;
-use crate::red_knot::semantic_index::ast_ids::{ScopeAstIdNode, ScopeExpressionId};
+use crate::semantic_index::ast_ids::{ScopeAstIdNode, ScopeExpressionId};
-use crate::red_knot::semantic_index::definition::{
+use crate::semantic_index::definition::{Definition, ImportDefinition, ImportFromDefinition};
-    Definition, ImportDefinition, ImportFromDefinition,
+use crate::semantic_index::symbol::{FileScopeId, ScopeId, ScopeKind, ScopedSymbolId, SymbolTable};
-};
+use crate::semantic_index::{symbol_table, ChildrenIter, SemanticIndex};
-use crate::red_knot::semantic_index::symbol::{
+use crate::types::{
-    FileScopeId, ScopeId, ScopeKind, ScopedSymbolId, SymbolTable,
-};
-use crate::red_knot::semantic_index::{symbol_table, ChildrenIter, SemanticIndex};
-use crate::red_knot::types::{
     ClassType, FunctionType, IntersectionType, ModuleType, ScopedClassTypeId, ScopedFunctionTypeId,
     ScopedIntersectionTypeId, ScopedUnionTypeId, Type, TypeId, TypingContext, UnionType,
     UnionTypeBuilder,
@@ -700,7 +696,7 @@ mod tests {
     use crate::db::tests::TestDb;
     use crate::module::resolver::{set_module_resolution_settings, ModuleResolutionSettings};
     use crate::name::Name;
-    use crate::red_knot::types::{public_symbol_ty_by_name, Type, TypingContext};
+    use crate::types::{public_symbol_ty_by_name, Type, TypingContext};
 
     fn setup_db() -> TestDb {
         let mut db = TestDb::new();

crates/ruff_python_semantic/Cargo.toml

@@ -11,7 +11,6 @@ repository = { workspace = true }
 license = { workspace = true }
 
 [dependencies]
-ruff_db = { workspace = true }
 ruff_index = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_stdlib = { workspace = true }
@@ -20,21 +19,11 @@ ruff_text_size = { workspace = true }
 
 bitflags = { workspace = true }
 is-macro = { workspace = true }
-indexmap = { workspace = true, optional = true }
-salsa = { workspace = true, optional = true }
-smallvec = { workspace = true, optional = true }
-smol_str = { workspace = true }
-tracing = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
-hashbrown = { workspace = true, optional = true }
 
 [dev-dependencies]
-anyhow = { workspace = true }
 ruff_python_parser = { workspace = true }
-tempfile = { workspace = true }
 
 [lints]
 workspace = true
 
-[features]
-red_knot = ["dep:salsa", "dep:tracing", "dep:hashbrown", "dep:smallvec", "dep:indexmap"]

crates/ruff_python_semantic/src/lib.rs

@@ -2,17 +2,10 @@ pub mod analyze;
 mod binding;
 mod branches;
 mod context;
-#[cfg(feature = "red_knot")]
-mod db;
 mod definition;
 mod globals;
 mod model;
-#[cfg(feature = "red_knot")]
-pub mod module;
-pub mod name;
 mod nodes;
-#[cfg(feature = "red_knot")]
-pub mod red_knot;
 mod reference;
 mod scope;
 mod star_import;
@@ -27,6 +20,3 @@ pub use nodes::*;
 pub use reference::*;
 pub use scope::*;
 pub use star_import::*;
-
-#[cfg(feature = "red_knot")]
-pub use db::{Db, Jar};