[red-knot] Merge the semantic and module-resolver crates (#12751)

This commit is contained in:
Alex Waygood 2024-08-08 15:34:11 +01:00 committed by GitHub
parent 33e9a6a54e
commit f1de08c2a0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
610 changed files with 91 additions and 274 deletions

View file

@ -37,13 +37,13 @@ jobs:
- name: Sync typeshed
id: sync
run: |
rm -rf ruff/crates/red_knot_module_resolver/vendor/typeshed
mkdir ruff/crates/red_knot_module_resolver/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_module_resolver/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_module_resolver/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_module_resolver/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_module_resolver/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_module_resolver/vendor/typeshed/source_commit.txt
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
- name: Commit the changes
id: commit
if: ${{ steps.sync.outcome == 'success' }}

View file

@ -2,15 +2,15 @@ fail_fast: true
exclude: |
(?x)^(
crates/red_knot_module_resolver/vendor/.*|
crates/red_knot_python_semantic/vendor/.*|
crates/red_knot_workspace/resources/.*|
crates/ruff_linter/resources/.*|
crates/ruff_linter/src/rules/.*/snapshots/.*|
crates/ruff/resources/.*|
crates/ruff_python_formatter/resources/.*|
crates/ruff_python_formatter/tests/snapshots/.*|
crates/ruff_python_resolver/resources/.*|
crates/ruff_python_resolver/tests/snapshots/.*|
crates/red_knot_workspace/resources/.*
crates/ruff_python_resolver/tests/snapshots/.*
)$
repos:

33
Cargo.lock generated
View file

@ -1868,7 +1868,7 @@ dependencies = [
"ctrlc",
"filetime",
"rayon",
"red_knot_module_resolver",
"red_knot_python_semantic",
"red_knot_server",
"red_knot_workspace",
"ruff_db",
@ -1881,17 +1881,25 @@ dependencies = [
]
[[package]]
name = "red_knot_module_resolver"
name = "red_knot_python_semantic"
version = "0.0.0"
dependencies = [
"anyhow",
"bitflags 2.6.0",
"camino",
"compact_str",
"countme",
"hashbrown",
"insta",
"once_cell",
"ordermap",
"path-slash",
"ruff_db",
"ruff_index",
"ruff_python_ast",
"ruff_python_parser",
"ruff_python_stdlib",
"ruff_text_size",
"rustc-hash 2.0.0",
"salsa",
"tempfile",
@ -1900,26 +1908,6 @@ dependencies = [
"zip",
]
[[package]]
name = "red_knot_python_semantic"
version = "0.0.0"
dependencies = [
"anyhow",
"bitflags 2.6.0",
"countme",
"hashbrown",
"ordermap",
"red_knot_module_resolver",
"ruff_db",
"ruff_index",
"ruff_python_ast",
"ruff_python_parser",
"ruff_text_size",
"rustc-hash 2.0.0",
"salsa",
"tracing",
]
[[package]]
name = "red_knot_server"
version = "0.0.0"
@ -1967,7 +1955,6 @@ dependencies = [
"anyhow",
"crossbeam",
"notify",
"red_knot_module_resolver",
"red_knot_python_semantic",
"ruff_cache",
"ruff_db",

View file

@ -35,7 +35,6 @@ ruff_source_file = { path = "crates/ruff_source_file" }
ruff_text_size = { path = "crates/ruff_text_size" }
ruff_workspace = { path = "crates/ruff_workspace" }
red_knot_module_resolver = { path = "crates/red_knot_module_resolver" }
red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
red_knot_server = { path = "crates/red_knot_server" }
red_knot_workspace = { path = "crates/red_knot_workspace" }

View file

@ -1,6 +1,6 @@
[files]
# https://github.com/crate-ci/typos/issues/868
extend-exclude = ["crates/red_knot_module_resolver/vendor/**/*", "**/resources/**/*", "**/snapshots/**/*"]
extend-exclude = ["crates/red_knot_python_semantic/vendor/**/*", "**/resources/**/*", "**/snapshots/**/*"]
[default.extend-words]
"arange" = "arange" # e.g. `numpy.arange`

View file

@ -12,7 +12,7 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
red_knot_module_resolver = { workspace = true }
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_server = { workspace = true }

View file

@ -6,7 +6,7 @@ use std::time::Duration;
use anyhow::{anyhow, Context};
use salsa::Setter;
use red_knot_module_resolver::{resolve_module, ModuleName};
use red_knot_python_semantic::{resolve_module, ModuleName};
use red_knot_workspace::db::RootDatabase;
use red_knot_workspace::watch;
use red_knot_workspace::watch::{directory_watcher, WorkspaceWatcher};

View file

@ -1,39 +0,0 @@
[package]
name = "red_knot_module_resolver"
version = "0.0.0"
publish = false
authors = { workspace = true }
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
[dependencies]
ruff_db = { workspace = true }
ruff_python_stdlib = { workspace = true }
compact_str = { workspace = true }
camino = { workspace = true }
once_cell = { workspace = true }
rustc-hash = { workspace = true }
salsa = { workspace = true }
tracing = { workspace = true }
zip = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true, features = ["zstd", "deflate"] }
[dev-dependencies]
ruff_db = { workspace = true, features = ["os"] }
anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
[lints]
workspace = true

View file

@ -1,104 +0,0 @@
use ruff_db::Upcast;
#[salsa::db]
pub trait Db: ruff_db::Db + Upcast<dyn ruff_db::Db> {}
#[cfg(test)]
pub(crate) mod tests {
use std::sync;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
use crate::vendored_typeshed_stubs;
use super::*;
#[salsa::db]
pub(crate) struct TestDb {
storage: salsa::Storage<Self>,
system: TestSystem,
vendored: VendoredFileSystem,
files: Files,
events: sync::Arc<sync::Mutex<Vec<salsa::Event>>>,
}
impl TestDb {
pub(crate) fn new() -> Self {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
events: sync::Arc::default(),
files: Files::default(),
}
}
/// Takes the salsa events.
///
/// ## Panics
/// If there are any pending salsa snapshots.
pub(crate) fn take_salsa_events(&mut self) -> Vec<salsa::Event> {
let inner = sync::Arc::get_mut(&mut self.events).expect("no pending salsa snapshots");
let events = inner.get_mut().unwrap();
std::mem::take(&mut *events)
}
/// Clears the salsa events.
///
/// ## Panics
/// If there are any pending salsa snapshots.
pub(crate) fn clear_salsa_events(&mut self) {
self.take_salsa_events();
}
}
impl Upcast<dyn ruff_db::Db> for TestDb {
fn upcast(&self) -> &(dyn ruff_db::Db + 'static) {
self
}
fn upcast_mut(&mut self) -> &mut (dyn ruff_db::Db + 'static) {
self
}
}
#[salsa::db]
impl ruff_db::Db for TestDb {
fn vendored(&self) -> &VendoredFileSystem {
&self.vendored
}
fn system(&self) -> &dyn ruff_db::system::System {
&self.system
}
fn files(&self) -> &Files {
&self.files
}
}
#[salsa::db]
impl Db for TestDb {}
impl DbWithTestSystem for TestDb {
fn test_system(&self) -> &TestSystem {
&self.system
}
fn test_system_mut(&mut self) -> &mut TestSystem {
&mut self.system
}
}
#[salsa::db]
impl salsa::Database for TestDb {
fn salsa_event(&self, event: &dyn Fn() -> salsa::Event) {
let event = event();
tracing::trace!("event: {event:?}");
let mut events = self.events.lock().unwrap();
events.push(event);
}
}
}

View file

@ -11,22 +11,34 @@ repository = { workspace = true }
license = { workspace = true }
[dependencies]
red_knot_module_resolver = { workspace = true }
ruff_db = { workspace = true }
ruff_index = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_python_stdlib = { workspace = true }
ruff_text_size = { workspace = true }
bitflags = { workspace = true }
camino = { workspace = true }
compact_str = { workspace = true }
countme = { workspace = true }
once_cell = { workspace = true }
ordermap = { workspace = true }
salsa = { workspace = true }
tracing = { workspace = true }
rustc-hash = { workspace = true }
hashbrown = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true, features = ["zstd", "deflate"] }
[dev-dependencies]
anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true }
ruff_python_parser = { workspace = true }
[lints]

View file

@ -1,9 +1,9 @@
# Red Knot
A work-in-progress multifile module resolver for Ruff.
Semantic analysis for the red-knot project.
## Vendored types for the stdlib
This crate vendors [typeshed](https://github.com/python/typeshed)'s stubs for the standard library. The vendored stubs can be found in `crates/red_knot_module_resolver/vendor/typeshed`. The file `crates/red_knot_module_resolver/vendor/typeshed/source_commit.txt` tells you the typeshed commit that our vendored stdlib stubs currently correspond to.
This crate vendors [typeshed](https://github.com/python/typeshed)'s stubs for the standard library. The vendored stubs can be found in `crates/red_knot_python_semantic/vendor/typeshed`. The file `crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt` tells you the typeshed commit that our vendored stdlib stubs currently correspond to.
The typeshed stubs are updated every two weeks via an automated PR using the `sync_typeshed.yaml` workflow in the `.github/workflows` directory. This workflow can also be triggered at any time via [workflow dispatch](https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow#running-a-workflow).

View file

@ -3,7 +3,7 @@
//!
//! This script should be automatically run at build time
//! whenever the script itself changes, or whenever any files
//! in `crates/red_knot_module_resolver/vendor/typeshed` change.
//! in `crates/red_knot_python_semantic/vendor/typeshed` change.
use std::fs::File;
use std::path::Path;

View file

@ -1,5 +1,5 @@
use red_knot_module_resolver::{resolve_module, ModuleName};
use crate::module_name::ModuleName;
use crate::module_resolver::resolve_module;
use crate::semantic_index::global_scope;
use crate::semantic_index::symbol::ScopeId;
use crate::Db;
@ -11,6 +11,6 @@ use crate::Db;
pub(crate) fn builtins_scope(db: &dyn Db) -> Option<ScopeId<'_>> {
let builtins_name =
ModuleName::new_static("builtins").expect("Expected 'builtins' to be a valid module name");
let builtins_file = resolve_module(db.upcast(), builtins_name)?.file();
let builtins_file = resolve_module(db, builtins_name)?.file();
Some(global_scope(db, builtins_file))
}

View file

@ -1,15 +1,14 @@
use red_knot_module_resolver::Db as ResolverDb;
use ruff_db::Upcast;
use ruff_db::{Db as SourceDb, Upcast};
/// Database giving access to semantic information about a Python program.
#[salsa::db]
pub trait Db: ResolverDb + Upcast<dyn ResolverDb> {}
pub trait Db: SourceDb + Upcast<dyn SourceDb> {}
#[cfg(test)]
pub(crate) mod tests {
use std::sync::Arc;
use red_knot_module_resolver::{vendored_typeshed_stubs, Db as ResolverDb};
use crate::module_resolver::vendored_typeshed_stubs;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
@ -91,18 +90,6 @@ pub(crate) mod tests {
}
}
impl Upcast<dyn ResolverDb> for TestDb {
fn upcast(&self) -> &(dyn ResolverDb + 'static) {
self
}
fn upcast_mut(&mut self) -> &mut (dyn ResolverDb + 'static) {
self
}
}
#[salsa::db]
impl red_knot_module_resolver::Db for TestDb {}
#[salsa::db]
impl Db for TestDb {}

View file

@ -3,11 +3,15 @@ use std::hash::BuildHasherDefault;
use rustc_hash::FxHasher;
pub use db::Db;
pub use module_name::ModuleName;
pub use module_resolver::{resolve_module, system_module_search_paths, vendored_typeshed_stubs};
pub use semantic_model::{HasTy, SemanticModel};
pub mod ast_node_ref;
mod builtins;
mod db;
mod module_name;
mod module_resolver;
mod node_key;
pub mod semantic_index;
mod semantic_model;

View file

@ -42,7 +42,7 @@ impl ModuleName {
/// ## Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar").as_deref(), Some("foo.bar"));
/// assert_eq!(ModuleName::new_static(""), None);
@ -68,7 +68,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::<Vec<_>>(), vec!["foo", "bar", "baz"]);
/// ```
@ -82,7 +82,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar").unwrap().parent(), Some(ModuleName::new_static("foo").unwrap()));
/// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap()));
@ -101,7 +101,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert!(ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap()));
///
@ -133,7 +133,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(&*ModuleName::from_components(["a"]).unwrap(), "a");
/// assert_eq!(&*ModuleName::from_components(["a", "b"]).unwrap(), "a.b");

View file

@ -1,19 +1,14 @@
use std::iter::FusedIterator;
pub use db::Db;
pub use module::{Module, ModuleKind};
pub use module_name::ModuleName;
pub(crate) use module::Module;
pub use resolver::resolve_module;
use ruff_db::system::SystemPath;
pub use typeshed::{
vendored_typeshed_stubs, TypeshedVersionsParseError, TypeshedVersionsParseErrorKind,
};
pub use typeshed::vendored_typeshed_stubs;
use crate::resolver::{module_resolution_settings, SearchPathIterator};
use crate::Db;
use resolver::{module_resolution_settings, SearchPathIterator};
mod db;
mod module;
mod module_name;
mod path;
mod resolver;
mod state;

View file

@ -3,8 +3,8 @@ use std::sync::Arc;
use ruff_db::files::File;
use super::path::SearchPath;
use crate::module_name::ModuleName;
use crate::path::SearchPath;
/// Representation of a Python module.
#[derive(Clone, PartialEq, Eq)]

View file

@ -11,8 +11,9 @@ use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::state::ResolverState;
use crate::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
use super::state::ResolverState;
use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
/// A path that points to a Python module.
///
@ -427,7 +428,7 @@ impl SearchPath {
SearchPathValidationError::VersionsIsADirectory(typeshed)
}
})?;
crate::typeshed::parse_typeshed_versions(db, typeshed_versions)
super::typeshed::parse_typeshed_versions(db, typeshed_versions)
.as_ref()
.map_err(|validation_error| {
SearchPathValidationError::VersionsParseError(validation_error.clone())
@ -623,7 +624,8 @@ mod tests {
use ruff_db::Db;
use crate::db::tests::TestDb;
use crate::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use super::*;

View file

@ -8,10 +8,11 @@ use ruff_db::vendored::VendoredPath;
use rustc_hash::{FxBuildHasher, FxHashSet};
use crate::db::Db;
use crate::module::{Module, ModuleKind};
use crate::module_name::ModuleName;
use crate::path::{ModulePath, SearchPath, SearchPathValidationError};
use crate::state::ResolverState;
use super::module::{Module, ModuleKind};
use super::path::{ModulePath, SearchPath, SearchPathValidationError};
use super::state::ResolverState;
/// Resolves a module name to a module.
pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option<Module> {
@ -619,9 +620,9 @@ mod tests {
use ruff_db::Db;
use crate::db::tests::TestDb;
use crate::module::ModuleKind;
use crate::module_name::ModuleName;
use crate::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::module_resolver::module::ModuleKind;
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use super::*;

View file

@ -1,8 +1,8 @@
use ruff_db::program::TargetVersion;
use ruff_db::vendored::VendoredFileSystem;
use super::typeshed::LazyTypeshedVersions;
use crate::db::Db;
use crate::typeshed::LazyTypeshedVersions;
pub(crate) struct ResolverState<'db> {
pub(crate) db: &'db dyn Db,

View file

@ -1,8 +1,8 @@
pub use self::vendored::vendored_typeshed_stubs;
pub(crate) use self::versions::{
parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsQueryResult,
pub(super) use self::versions::{
parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsParseError,
TypeshedVersionsQueryResult,
};
pub use self::versions::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind};
mod vendored;
mod versions;

View file

@ -90,7 +90,7 @@ static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
});
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct TypeshedVersionsParseError {
pub(crate) struct TypeshedVersionsParseError {
line_number: Option<NonZeroU16>,
reason: TypeshedVersionsParseErrorKind,
}
@ -123,7 +123,7 @@ impl std::error::Error for TypeshedVersionsParseError {
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum TypeshedVersionsParseErrorKind {
pub(super) enum TypeshedVersionsParseErrorKind {
TooManyLines(NonZeroUsize),
UnexpectedNumberOfColons,
InvalidModuleName(String),
@ -505,7 +505,7 @@ mod tests {
#[test]
fn typeshed_versions_consistent_with_vendored_stubs() {
const VERSIONS_DATA: &str = include_str!("../../vendor/typeshed/stdlib/VERSIONS");
const VERSIONS_DATA: &str = include_str!("../../../vendor/typeshed/stdlib/VERSIONS");
let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
let vendored_typeshed_versions = TypeshedVersions::from_str(VERSIONS_DATA).unwrap();

View file

@ -1,8 +1,9 @@
use red_knot_module_resolver::{resolve_module, Module, ModuleName};
use ruff_db::files::File;
use ruff_python_ast as ast;
use ruff_python_ast::{Expr, ExpressionRef, StmtClassDef};
use crate::module_name::ModuleName;
use crate::module_resolver::{resolve_module, Module};
use crate::semantic_index::ast_ids::HasScopedAstId;
use crate::semantic_index::semantic_index;
use crate::types::{definition_ty, global_symbol_ty_by_name, infer_scope_types, Type};
@ -25,7 +26,7 @@ impl<'db> SemanticModel<'db> {
}
pub fn resolve_module(&self, module_name: ModuleName) -> Option<Module> {
resolve_module(self.db.upcast(), module_name)
resolve_module(self.db, module_name)
}
pub fn global_symbol_ty(&self, module: &Module, symbol_name: &str) -> Type<'db> {

View file

@ -24,13 +24,14 @@ use rustc_hash::FxHashMap;
use salsa;
use salsa::plumbing::AsId;
use red_knot_module_resolver::{resolve_module, ModuleName};
use ruff_db::files::File;
use ruff_db::parsed::parsed_module;
use ruff_python_ast as ast;
use ruff_python_ast::{ExprContext, TypeParams};
use crate::builtins::builtins_scope;
use crate::module_name::ModuleName;
use crate::module_resolver::resolve_module;
use crate::semantic_index::ast_ids::{HasScopedAstId, HasScopedUseId, ScopedExpressionId};
use crate::semantic_index::definition::{Definition, DefinitionKind, DefinitionNodeKey};
use crate::semantic_index::expression::Expression;
@ -840,9 +841,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
fn module_ty_from_name(&self, name: &ast::Identifier) -> Type<'db> {
let module_name = ModuleName::new(&name.id);
let module =
module_name.and_then(|module_name| resolve_module(self.db.upcast(), module_name));
let module = ModuleName::new(&name.id).and_then(|name| resolve_module(self.db, name));
module
.map(|module| Type::Module(module.file()))
.unwrap_or(Type::Unbound)

Some files were not shown because too many files have changed in this diff Show more