mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-02 01:42:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Eagerly validate typeshed versions (#12786)

Browse source
This commit is contained in:
Micha Reiser 2024-08-21 17:49:53 +02:00 • committed by GitHub
parent f873d2ac12
commit dce87c21fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 676 additions and 498 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/red_knot_python_semantic/Cargo.toml
View file

@ -26,6 +26,7 @@ countme = { workspace = true }
once_cell = { workspace = true }
ordermap = { workspace = true }
salsa = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
rustc-hash = { workspace = true }
hashbrown = { workspace = true }

3
crates/red_knot_python_semantic/src/lib.rs
View file

@ -5,7 +5,7 @@ use rustc_hash::FxHasher;
pub use db::Db;
pub use module_name::ModuleName;
pub use module_resolver::{resolve_module, system_module_search_paths, vendored_typeshed_stubs};
pub use program::{Program, ProgramSettings, SearchPathSettings};
pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
pub use python_version::PythonVersion;
pub use semantic_model::{HasTy, SemanticModel};
@ -19,6 +19,7 @@ mod program;
mod python_version;
pub mod semantic_index;
mod semantic_model;
pub(crate) mod site_packages;
pub mod types;
type FxOrderSet<V> = ordermap::set::OrderSet<V, BuildHasherDefault<FxHasher>>;

1
crates/red_knot_python_semantic/src/module_resolver/mod.rs
View file

@ -13,7 +13,6 @@ use resolver::SearchPathIterator;
mod module;
mod path;
mod resolver;
mod state;
mod typeshed;
#[cfg(test)]

148
crates/red_knot_python_semantic/src/module_resolver/path.rs
View file

@ -9,11 +9,11 @@ use ruff_db::files::{system_path_to_file, vendored_path_to_file, File, FileError
use ruff_db::system::{System, SystemPath, SystemPathBuf};
use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
use super::typeshed::{typeshed_versions, TypeshedVersionsParseError, TypeshedVersionsQueryResult};
use crate::db::Db;
use crate::module_name::ModuleName;
use super::state::ResolverState;
use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
use crate::module_resolver::resolver::ResolverContext;
use crate::site_packages::SitePackagesDiscoveryError;
/// A path that points to a Python module.
///
@ -60,7 +60,7 @@ impl ModulePath {
}
#[must_use]
pub(crate) fn is_directory(&self, resolver: &ResolverState) -> bool {
pub(super) fn is_directory(&self, resolver: &ResolverContext) -> bool {
let ModulePath {
search_path,
relative_path,
@ -74,7 +74,7 @@ impl ModulePath {
== Err(FileError::IsADirectory)
}
SearchPathInner::StandardLibraryCustom(stdlib_root) => {
match query_stdlib_version(Some(stdlib_root), relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => false,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => {
@ -84,7 +84,7 @@ impl ModulePath {
}
}
SearchPathInner::StandardLibraryVendored(stdlib_root) => {
match query_stdlib_version(None, relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => false,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => resolver
@ -96,7 +96,7 @@ impl ModulePath {
}
#[must_use]
pub(crate) fn is_regular_package(&self, resolver: &ResolverState) -> bool {
pub(super) fn is_regular_package(&self, resolver: &ResolverContext) -> bool {
let ModulePath {
search_path,
relative_path,
@ -113,7 +113,7 @@ impl ModulePath {
.is_ok()
}
SearchPathInner::StandardLibraryCustom(search_path) => {
match query_stdlib_version(Some(search_path), relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => false,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(
@ -124,7 +124,7 @@ impl ModulePath {
}
}
SearchPathInner::StandardLibraryVendored(search_path) => {
match query_stdlib_version(None, relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => false,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => resolver
@ -136,7 +136,7 @@ impl ModulePath {
}
#[must_use]
pub(crate) fn to_file(&self, resolver: &ResolverState) -> Option<File> {
pub(super) fn to_file(&self, resolver: &ResolverContext) -> Option<File> {
let db = resolver.db.upcast();
let ModulePath {
search_path,
@ -150,7 +150,7 @@ impl ModulePath {
system_path_to_file(db, search_path.join(relative_path)).ok()
}
SearchPathInner::StandardLibraryCustom(stdlib_root) => {
match query_stdlib_version(Some(stdlib_root), relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => None,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => {
@ -159,7 +159,7 @@ impl ModulePath {
}
}
SearchPathInner::StandardLibraryVendored(stdlib_root) => {
match query_stdlib_version(None, relative_path, resolver) {
match query_stdlib_version(relative_path, resolver) {
TypeshedVersionsQueryResult::DoesNotExist => None,
TypeshedVersionsQueryResult::Exists
| TypeshedVersionsQueryResult::MaybeExists => {
@ -273,19 +273,15 @@ fn stdlib_path_to_module_name(relative_path: &Utf8Path) -> Option<ModuleName> {
#[must_use]
fn query_stdlib_version(
custom_stdlib_root: Option<&SystemPath>,
relative_path: &Utf8Path,
resolver: &ResolverState,
context: &ResolverContext,
) -> TypeshedVersionsQueryResult {
let Some(module_name) = stdlib_path_to_module_name(relative_path) else {
return TypeshedVersionsQueryResult::DoesNotExist;
};
let ResolverState {
db,
typeshed_versions,
target_version,
} = resolver;
typeshed_versions.query_module(*db, &module_name, custom_stdlib_root, *target_version)
let ResolverContext { db, target_version } = context;
typeshed_versions(*db).query_module(&module_name, *target_version)
}
/// Enumeration describing the various ways in which validation of a search path might fail.
@ -293,7 +289,7 @@ fn query_stdlib_version(
/// If validation fails for a search path derived from the user settings,
/// a message must be displayed to the user,
/// as type checking cannot be done reliably in these circumstances.
#[derive(Debug, PartialEq, Eq)]
#[derive(Debug)]
pub(crate) enum SearchPathValidationError {
/// The path provided by the user was not a directory
NotADirectory(SystemPathBuf),
@ -304,18 +300,20 @@ pub(crate) enum SearchPathValidationError {
NoStdlibSubdirectory(SystemPathBuf),
/// The typeshed path provided by the user is a directory,
/// but no `stdlib/VERSIONS` file exists.
/// but `stdlib/VERSIONS` could not be read.
/// (This is only relevant for stdlib search paths.)
NoVersionsFile(SystemPathBuf),
/// `stdlib/VERSIONS` is a directory.
/// (This is only relevant for stdlib search paths.)
VersionsIsADirectory(SystemPathBuf),
FailedToReadVersionsFile {
path: SystemPathBuf,
error: std::io::Error,
},
/// The path provided by the user is a directory,
/// and a `stdlib/VERSIONS` file exists, but it fails to parse.
/// (This is only relevant for stdlib search paths.)
VersionsParseError(TypeshedVersionsParseError),
/// Failed to discover the site-packages for the configured virtual environment.
SitePackagesDiscovery(SitePackagesDiscoveryError),
}
impl fmt::Display for SearchPathValidationError {
@ -325,9 +323,16 @@ impl fmt::Display for SearchPathValidationError {
Self::NoStdlibSubdirectory(path) => {
write!(f, "The directory at {path} has no `stdlib/` subdirectory")
}
Self::NoVersionsFile(path) => write!(f, "Expected a file at {path}/stdlib/VERSIONS"),
Self::VersionsIsADirectory(path) => write!(f, "{path}/stdlib/VERSIONS is a directory."),
Self::FailedToReadVersionsFile { path, error } => {
write!(
f,
"Failed to read the custom typeshed versions file '{path}': {error}"
)
}
Self::VersionsParseError(underlying_error) => underlying_error.fmt(f),
SearchPathValidationError::SitePackagesDiscovery(error) => {
write!(f, "Failed to discover the site-packages directory: {error}")
}
}
}
}
@ -342,6 +347,18 @@ impl std::error::Error for SearchPathValidationError {
}
}
impl From<TypeshedVersionsParseError> for SearchPathValidationError {
fn from(value: TypeshedVersionsParseError) -> Self {
Self::VersionsParseError(value)
}
}
impl From<SitePackagesDiscoveryError> for SearchPathValidationError {
fn from(value: SitePackagesDiscoveryError) -> Self {
Self::SitePackagesDiscovery(value)
}
}
type SearchPathResult<T> = Result<T, SearchPathValidationError>;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@ -384,11 +401,10 @@ pub(crate) struct SearchPath(Arc<SearchPathInner>);
impl SearchPath {
fn directory_path(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<SystemPathBuf> {
let canonicalized = system.canonicalize_path(&root).unwrap_or(root);
if system.is_directory(&canonicalized) {
Ok(canonicalized)
if system.is_directory(&root) {
Ok(root)
} else {
Err(SearchPathValidationError::NotADirectory(canonicalized))
Err(SearchPathValidationError::NotADirectory(root))
}
}
@ -407,32 +423,22 @@ impl SearchPath {
}
/// Create a new standard-library search path pointing to a custom directory on disk
pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: SystemPathBuf) -> SearchPathResult<Self> {
pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: &SystemPath) -> SearchPathResult<Self> {
let system = db.system();
if !system.is_directory(&typeshed) {
if !system.is_directory(typeshed) {
return Err(SearchPathValidationError::NotADirectory(
typeshed.to_path_buf(),
));
}
let stdlib =
Self::directory_path(system, typeshed.join("stdlib")).map_err(|err| match err {
SearchPathValidationError::NotADirectory(path) => {
SearchPathValidationError::NoStdlibSubdirectory(path)
SearchPathValidationError::NotADirectory(_) => {
SearchPathValidationError::NoStdlibSubdirectory(typeshed.to_path_buf())
}
err => err,
})?;
let typeshed_versions =
system_path_to_file(db.upcast(), stdlib.join("VERSIONS")).map_err(|err| match err {
FileError::NotFound => SearchPathValidationError::NoVersionsFile(typeshed),
FileError::IsADirectory => {
SearchPathValidationError::VersionsIsADirectory(typeshed)
}
})?;
super::typeshed::parse_typeshed_versions(db, typeshed_versions)
.as_ref()
.map_err(|validation_error| {
SearchPathValidationError::VersionsParseError(validation_error.clone())
})?;
Ok(Self(Arc::new(SearchPathInner::StandardLibraryCustom(
stdlib,
))))
@ -623,11 +629,11 @@ mod tests {
use ruff_db::Db;
use crate::db::tests::TestDb;
use super::*;
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::python_version::PythonVersion;
use super::*;
impl ModulePath {
#[must_use]
fn join(&self, component: &str) -> ModulePath {
@ -638,15 +644,6 @@ mod tests {
}
impl SearchPath {
#[must_use]
pub(crate) fn is_stdlib_search_path(&self) -> bool {
matches!(
&*self.0,
SearchPathInner::StandardLibraryCustom(_)
| SearchPathInner::StandardLibraryVendored(_)
)
}
fn join(&self, component: &str) -> ModulePath {
self.to_module_path().join(component)
}
@ -661,7 +658,7 @@ mod tests {
.build();
assert_eq!(
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.with_py_extension(),
@ -669,7 +666,7 @@ mod tests {
);
assert_eq!(
&SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
&SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
.unwrap()
.join("foo")
.with_pyi_extension(),
@ -780,7 +777,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(MockedTypeshed::default())
.build();
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.push("bar.py");
@ -792,7 +789,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(MockedTypeshed::default())
.build();
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.push("bar.rs");
@ -824,7 +821,7 @@ mod tests {
.with_custom_typeshed(MockedTypeshed::default())
.build();
let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap();
let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
// Must have a `.pyi` extension or no extension:
let bad_absolute_path = SystemPath::new("foo/stdlib/x.py");
@ -872,8 +869,7 @@ mod tests {
.with_custom_typeshed(typeshed)
.with_target_version(target_version)
.build();
let stdlib =
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap();
let stdlib = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
(db, stdlib)
}
@ -898,7 +894,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let asyncio_regular_package = stdlib_path.join("asyncio");
assert!(asyncio_regular_package.is_directory(&resolver));
@ -926,7 +922,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let xml_namespace_package = stdlib_path.join("xml");
assert!(xml_namespace_package.is_directory(&resolver));
@ -948,7 +944,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let functools_module = stdlib_path.join("functools.pyi");
assert!(functools_module.to_file(&resolver).is_some());
@ -964,7 +960,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let collections_regular_package = stdlib_path.join("collections");
assert_eq!(collections_regular_package.to_file(&resolver), None);
@ -980,7 +976,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let importlib_namespace_package = stdlib_path.join("importlib");
assert_eq!(importlib_namespace_package.to_file(&resolver), None);
@ -1001,7 +997,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let resolver = ResolverContext::new(&db, PythonVersion::PY38);
let non_existent = stdlib_path.join("doesnt_even_exist");
assert_eq!(non_existent.to_file(&resolver), None);
@ -1029,7 +1025,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
let resolver = ResolverContext::new(&db, PythonVersion::PY39);
// Since we've set the target version to Py39,
// `collections` should now exist as a directory, according to VERSIONS...
@ -1058,7 +1054,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
let resolver = ResolverContext::new(&db, PythonVersion::PY39);
// The `importlib` directory now also exists
let importlib_namespace_package = stdlib_path.join("importlib");
@ -1082,7 +1078,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
let resolver = ResolverContext::new(&db, PythonVersion::PY39);
// The `xml` package no longer exists on py39:
let xml_namespace_package = stdlib_path.join("xml");

171
crates/red_knot_python_semantic/src/module_resolver/resolver.rs
View file

@ -1,19 +1,19 @@
use rustc_hash::{FxBuildHasher, FxHashSet};
use std::borrow::Cow;
use std::iter::FusedIterator;
use rustc_hash::{FxBuildHasher, FxHashSet};
use std::ops::Deref;
use ruff_db::files::{File, FilePath, FileRootKind};
use ruff_db::system::{DirectoryEntry, SystemPath, SystemPathBuf};
use ruff_db::vendored::VendoredPath;
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::{Program, SearchPathSettings};
use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
use ruff_db::vendored::{VendoredFileSystem, VendoredPath};
use super::module::{Module, ModuleKind};
use super::path::{ModulePath, SearchPath, SearchPathValidationError};
use super::state::ResolverState;
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions};
use crate::site_packages::VirtualEnvironment;
use crate::{Program, PythonVersion, SearchPathSettings, SitePackages};
/// Resolves a module name to a module.
pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option<Module> {
@ -122,7 +122,7 @@ pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator {
Program::get(db).search_paths(db).iter(db)
}
#[derive(Debug, PartialEq, Eq, Default)]
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct SearchPaths {
/// Search paths that have been statically determined purely from reading Ruff's configuration settings.
/// These shouldn't ever change unless the config settings themselves change.
@ -135,6 +135,8 @@ pub(crate) struct SearchPaths {
/// in terms of module-resolution priority until we've discovered the editable installs
/// for the first `site-packages` path
site_packages: Vec<SearchPath>,
typeshed_versions: ResolvedTypeshedVersions,
}
impl SearchPaths {
@ -146,8 +148,14 @@ impl SearchPaths {
/// [module resolution order]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering
pub(crate) fn from_settings(
db: &dyn Db,
settings: SearchPathSettings,
settings: &SearchPathSettings,
) -> Result<Self, SearchPathValidationError> {
fn canonicalize(path: &SystemPath, system: &dyn System) -> SystemPathBuf {
system
.canonicalize_path(path)
.unwrap_or_else(|_| path.to_path_buf())
}
let SearchPathSettings {
extra_paths,
src_root,
@ -161,45 +169,65 @@ impl SearchPaths {
let mut static_paths = vec![];
for path in extra_paths {
tracing::debug!("Adding static extra search-path '{path}'");
let path = canonicalize(path, system);
files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
tracing::debug!("Adding extra search-path '{path}'");
let search_path = SearchPath::extra(system, path)?;
files.try_add_root(
db.upcast(),
search_path.as_system_path().unwrap(),
FileRootKind::LibrarySearchPath,
);
static_paths.push(search_path);
static_paths.push(SearchPath::extra(system, path)?);
}
tracing::debug!("Adding first-party search path '{src_root}'");
static_paths.push(SearchPath::first_party(system, src_root)?);
static_paths.push(SearchPath::first_party(system, src_root.to_path_buf())?);
static_paths.push(if let Some(custom_typeshed) = custom_typeshed {
let (typeshed_versions, stdlib_path) = if let Some(custom_typeshed) = custom_typeshed {
let custom_typeshed = canonicalize(custom_typeshed, system);
tracing::debug!("Adding custom-stdlib search path '{custom_typeshed}'");
let search_path = SearchPath::custom_stdlib(db, custom_typeshed)?;
files.try_add_root(
db.upcast(),
search_path.as_system_path().unwrap(),
&custom_typeshed,
FileRootKind::LibrarySearchPath,
);
search_path
let versions_path = custom_typeshed.join("stdlib/VERSIONS");
let versions_content = system.read_to_string(&versions_path).map_err(|error| {
SearchPathValidationError::FailedToReadVersionsFile {
path: versions_path,
error,
}
})?;
let parsed: TypeshedVersions = versions_content.parse()?;
let search_path = SearchPath::custom_stdlib(db, &custom_typeshed)?;
(ResolvedTypeshedVersions::Custom(parsed), search_path)
} else {
SearchPath::vendored_stdlib()
});
tracing::debug!("Using vendored stdlib");
(
ResolvedTypeshedVersions::Vendored(vendored_typeshed_versions()),
SearchPath::vendored_stdlib(),
)
};
static_paths.push(stdlib_path);
let site_packages_paths = match site_packages_paths {
SitePackages::Derived { venv_path } => VirtualEnvironment::new(venv_path, system)
.and_then(|venv| venv.site_packages_directories(system))?,
SitePackages::Known(paths) => paths
.iter()
.map(|path| canonicalize(path, system))
.collect(),
};
let mut site_packages: Vec<_> = Vec::with_capacity(site_packages_paths.len());
for path in site_packages_paths {
tracing::debug!("Adding site-packages search path '{path}'");
let search_path = SearchPath::site_packages(system, path)?;
files.try_add_root(
db.upcast(),
search_path.as_system_path().unwrap(),
FileRootKind::LibrarySearchPath,
);
site_packages.push(search_path);
files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
site_packages.push(SearchPath::site_packages(system, path)?);
}
// TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step
@ -224,16 +252,48 @@ impl SearchPaths {
Ok(SearchPaths {
static_paths,
site_packages,
typeshed_versions,
})
}
pub(crate) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
pub(super) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
SearchPathIterator {
db,
static_paths: self.static_paths.iter(),
dynamic_paths: None,
}
}
pub(crate) fn custom_stdlib(&self) -> Option<&SystemPath> {
self.static_paths.iter().find_map(|search_path| {
if search_path.is_standard_library() {
search_path.as_system_path()
} else {
None
}
})
}
pub(super) fn typeshed_versions(&self) -> &TypeshedVersions {
&self.typeshed_versions
}
}
#[derive(Debug, PartialEq, Eq)]
enum ResolvedTypeshedVersions {
Vendored(&'static TypeshedVersions),
Custom(TypeshedVersions),
}
impl Deref for ResolvedTypeshedVersions {
type Target = TypeshedVersions;
fn deref(&self) -> &Self::Target {
match self {
ResolvedTypeshedVersions::Vendored(versions) => versions,
ResolvedTypeshedVersions::Custom(versions) => versions,
}
}
}
/// Collect all dynamic search paths. For each `site-packages` path:
@ -251,6 +311,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
let SearchPaths {
static_paths,
site_packages,
typeshed_versions: _,
} = Program::get(db).search_paths(db);
let mut dynamic_paths = Vec::new();
@ -315,12 +376,16 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
let installations = all_pth_files.iter().flat_map(PthFile::items);
for installation in installations {
let installation = system
.canonicalize_path(&installation)
.unwrap_or(installation);
if existing_paths.insert(Cow::Owned(installation.clone())) {
match SearchPath::editable(system, installation) {
match SearchPath::editable(system, installation.clone()) {
Ok(search_path) => {
tracing::debug!(
"Adding editable installation to module resolution path {path}",
path = search_path.as_system_path().unwrap()
path = installation
);
dynamic_paths.push(search_path);
}
@ -482,7 +547,7 @@ struct ModuleNameIngredient<'db> {
fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, ModuleKind)> {
let program = Program::get(db);
let target_version = program.target_version(db);
let resolver_state = ResolverState::new(db, target_version);
let resolver_state = ResolverContext::new(db, target_version);
let is_builtin_module =
ruff_python_stdlib::sys::is_builtin_module(target_version.minor, name.as_str());
@ -545,7 +610,7 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, Mod
fn resolve_package<'a, 'db, I>(
module_search_path: &SearchPath,
components: I,
resolver_state: &ResolverState<'db>,
resolver_state: &ResolverContext<'db>,
) -> Result<ResolvedPackage, PackageKind>
where
I: Iterator<Item = &'a str>,
@ -627,6 +692,21 @@ impl PackageKind {
}
}
pub(super) struct ResolverContext<'db> {
pub(super) db: &'db dyn Db,
pub(super) target_version: PythonVersion,
}
impl<'db> ResolverContext<'db> {
pub(super) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
Self { db, target_version }
}
pub(super) fn vendored(&self) -> &VendoredFileSystem {
self.db.vendored()
}
}
#[cfg(test)]
mod tests {
use ruff_db::files::{system_path_to_file, File, FilePath};
@ -781,7 +861,7 @@ mod tests {
"Search path for {module_name} was unexpectedly {search_path:?}"
);
assert!(
search_path.is_stdlib_search_path(),
search_path.is_standard_library(),
"Expected a stdlib search path, but got {search_path:?}"
);
}
@ -877,7 +957,7 @@ mod tests {
"Search path for {module_name} was unexpectedly {search_path:?}"
);
assert!(
search_path.is_stdlib_search_path(),
search_path.is_standard_library(),
"Expected a stdlib search path, but got {search_path:?}"
);
}
@ -1194,13 +1274,13 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::PY38,
search_paths: SearchPathSettings {
extra_paths: vec![],
src_root: src.clone(),
custom_typeshed: Some(custom_typeshed.clone()),
site_packages: vec![site_packages],
site_packages: SitePackages::Known(vec![site_packages]),
},
},
)
@ -1699,13 +1779,16 @@ not_a_directory
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_root: SystemPathBuf::from("/src"),
custom_typeshed: None,
site_packages: vec![venv_site_packages, system_site_packages],
site_packages: SitePackages::Known(vec![
venv_site_packages,
system_site_packages,
]),
},
},
)

25
crates/red_knot_python_semantic/src/module_resolver/state.rs
View file

@ -1,25 +0,0 @@
use ruff_db::vendored::VendoredFileSystem;
use super::typeshed::LazyTypeshedVersions;
use crate::db::Db;
use crate::python_version::PythonVersion;
pub(crate) struct ResolverState<'db> {
pub(crate) db: &'db dyn Db,
pub(crate) typeshed_versions: LazyTypeshedVersions<'db>,
pub(crate) target_version: PythonVersion,
}
impl<'db> ResolverState<'db> {
pub(crate) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
Self {
db,
typeshed_versions: LazyTypeshedVersions::new(),
target_version,
}
}
pub(crate) fn vendored(&self) -> &VendoredFileSystem {
self.db.vendored()
}
}

16
crates/red_knot_python_semantic/src/module_resolver/testing.rs
View file

@ -4,7 +4,7 @@ use ruff_db::vendored::VendoredPathBuf;
use crate::db::tests::TestDb;
use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion;
use crate::ProgramSettings;
use crate::{ProgramSettings, SitePackages};
/// A test case for the module resolver.
///
@ -179,6 +179,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
first_party_files,
site_packages_files,
} = self;
TestCaseBuilder {
typeshed_option: typeshed,
target_version,
@ -195,6 +196,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
site_packages,
target_version,
} = self.with_custom_typeshed(MockedTypeshed::default()).build();
TestCase {
db,
src,
@ -223,13 +225,13 @@ impl TestCaseBuilder<MockedTypeshed> {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version,
search_paths: SearchPathSettings {
extra_paths: vec![],
src_root: src.clone(),
custom_typeshed: Some(typeshed.clone()),
site_packages: vec![site_packages.clone()],
site_packages: SitePackages::Known(vec![site_packages.clone()]),
},
},
)
@ -279,13 +281,11 @@ impl TestCaseBuilder<VendoredTypeshed> {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version,
search_paths: SearchPathSettings {
extra_paths: vec![],
src_root: src.clone(),
custom_typeshed: None,
site_packages: vec![site_packages.clone()],
site_packages: SitePackages::Known(vec![site_packages.clone()]),
..SearchPathSettings::new(src.clone())
},
},
)

2
crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs
View file

@ -1,6 +1,6 @@
pub use self::vendored::vendored_typeshed_stubs;
pub(super) use self::versions::{
parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsParseError,
typeshed_versions, vendored_typeshed_versions, TypeshedVersions, TypeshedVersionsParseError,
TypeshedVersionsQueryResult,
};

81
crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs
View file

@ -1,4 +1,3 @@
use std::cell::OnceCell;
use std::collections::BTreeMap;
use std::fmt;
use std::num::{NonZeroU16, NonZeroUsize};
@ -6,78 +5,12 @@ use std::ops::{RangeFrom, RangeInclusive};
use std::str::FromStr;
use once_cell::sync::Lazy;
use ruff_db::system::SystemPath;
use rustc_hash::FxHashMap;
use ruff_db::files::{system_path_to_file, File};
use super::vendored::vendored_typeshed_stubs;
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::python_version::PythonVersion;
#[derive(Debug)]
pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>);
impl<'db> LazyTypeshedVersions<'db> {
#[must_use]
pub(crate) fn new() -> Self {
Self(OnceCell::new())
}
/// Query whether a module exists at runtime in the stdlib on a certain Python version.
///
/// Simply probing whether a file exists in typeshed is insufficient for this question,
/// as a module in the stdlib may have been added in Python 3.10, but the typeshed stub
/// will still be available (either in a custom typeshed dir or in our vendored copy)
/// even if the user specified Python 3.8 as the target version.
///
/// For top-level modules and packages, the VERSIONS file can always provide an unambiguous answer
/// as to whether the module exists on the specified target version. However, VERSIONS does not
/// provide comprehensive information on all submodules, meaning that this method sometimes
/// returns [`TypeshedVersionsQueryResult::MaybeExists`].
/// See [`TypeshedVersionsQueryResult`] for more details.
#[must_use]
pub(crate) fn query_module(
&self,
db: &'db dyn Db,
module: &ModuleName,
stdlib_root: Option<&SystemPath>,
target_version: PythonVersion,
) -> TypeshedVersionsQueryResult {
let versions = self.0.get_or_init(|| {
let versions_path = if let Some(system_path) = stdlib_root {
system_path.join("VERSIONS")
} else {
return &VENDORED_VERSIONS;
};
let Ok(versions_file) = system_path_to_file(db.upcast(), &versions_path) else {
todo!(
"Still need to figure out how to handle VERSIONS files being deleted \
from custom typeshed directories! Expected a file to exist at {versions_path}"
)
};
// TODO(Alex/Micha): If VERSIONS is invalid,
// this should invalidate not just the specific module resolution we're currently attempting,
// but all type inference that depends on any standard-library types.
// Unwrapping here is not correct...
parse_typeshed_versions(db, versions_file).as_ref().unwrap()
});
versions.query_module(module, target_version)
}
}
#[salsa::tracked(return_ref)]
pub(crate) fn parse_typeshed_versions(
db: &dyn Db,
versions_file: File,
) -> Result<TypeshedVersions, TypeshedVersionsParseError> {
// TODO: Handle IO errors
let file_content = versions_file
.read_to_string(db.upcast())
.unwrap_or_default();
file_content.parse()
}
use crate::{Program, PythonVersion};
static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
TypeshedVersions::from_str(
@ -88,6 +21,14 @@ static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
.unwrap()
});
pub(crate) fn vendored_typeshed_versions() -> &'static TypeshedVersions {
&VENDORED_VERSIONS
}
pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions {
Program::get(db).search_paths(db).typeshed_versions()
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub(crate) struct TypeshedVersionsParseError {
line_number: Option<NonZeroU16>,
@ -174,7 +115,7 @@ impl TypeshedVersions {
}
#[must_use]
fn query_module(
pub(in crate::module_resolver) fn query_module(
&self,
module: &ModuleName,
target_version: PythonVersion,
@ -204,7 +145,7 @@ impl TypeshedVersions {
}
}
/// Possible answers [`LazyTypeshedVersions::query_module()`] could give to the question:
/// Possible answers [`TypeshedVersions::query_module()`] could give to the question:
/// "Does this module exist in the stdlib at runtime on a certain target version?"
#[derive(Debug, Copy, PartialEq, Eq, Clone, Hash)]
pub(crate) enum TypeshedVersionsQueryResult {

42
crates/red_knot_python_semantic/src/program.rs
View file

@ -3,7 +3,7 @@ use anyhow::Context;
use salsa::Durability;
use salsa::Setter;
use ruff_db::system::SystemPathBuf;
use ruff_db::system::{SystemPath, SystemPathBuf};
use crate::module_resolver::SearchPaths;
use crate::Db;
@ -12,13 +12,12 @@ use crate::Db;
pub struct Program {
pub target_version: PythonVersion,
#[default]
#[return_ref]
pub(crate) search_paths: SearchPaths,
}
impl Program {
pub fn from_settings(db: &dyn Db, settings: ProgramSettings) -> anyhow::Result<Self> {
pub fn from_settings(db: &dyn Db, settings: &ProgramSettings) -> anyhow::Result<Self> {
let ProgramSettings {
target_version,
search_paths,
@ -29,16 +28,15 @@ impl Program {
let search_paths = SearchPaths::from_settings(db, search_paths)
.with_context(|| "Invalid search path settings")?;
Ok(Program::builder(settings.target_version)
Ok(Program::builder(settings.target_version, search_paths)
.durability(Durability::HIGH)
.search_paths(search_paths)
.new(db))
}
pub fn update_search_paths(
&self,
self,
db: &mut dyn Db,
search_path_settings: SearchPathSettings,
search_path_settings: &SearchPathSettings,
) -> anyhow::Result<()> {
let search_paths = SearchPaths::from_settings(db, search_path_settings)?;
@ -49,16 +47,20 @@ impl Program {
Ok(())
}
pub fn custom_stdlib_search_path(self, db: &dyn Db) -> Option<&SystemPath> {
self.search_paths(db).custom_stdlib()
}
}
#[derive(Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ProgramSettings {
pub target_version: PythonVersion,
pub search_paths: SearchPathSettings,
}
/// Configures the search paths for module resolution.
#[derive(Eq, PartialEq, Debug, Clone, Default)]
#[derive(Eq, PartialEq, Debug, Clone)]
pub struct SearchPathSettings {
/// List of user-provided paths that should take first priority in the module resolution.
/// Examples in other type checkers are mypy's MYPYPATH environment variable,
@ -74,5 +76,25 @@ pub struct SearchPathSettings {
pub custom_typeshed: Option<SystemPathBuf>,
/// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed.
pub site_packages: Vec<SystemPathBuf>,
pub site_packages: SitePackages,
}
impl SearchPathSettings {
pub fn new(src_root: SystemPathBuf) -> Self {
Self {
src_root,
extra_paths: vec![],
custom_typeshed: None,
site_packages: SitePackages::Known(vec![]),
}
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum SitePackages {
Derived {
venv_path: SystemPathBuf,
},
/// Resolved site packages paths
Known(Vec<SystemPathBuf>),
}

9
crates/red_knot_python_semantic/src/semantic_model.rs
View file

@ -184,14 +184,9 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_root: SystemPathBuf::from("/src"),
site_packages: vec![],
custom_typeshed: None,
},
search_paths: SearchPathSettings::new(SystemPathBuf::from("/src")),
},
)?;

842
crates/red_knot_python_semantic/src/site_packages.rs Normal file
View file

@ -0,0 +1,842 @@
//! Utilities for finding the `site-packages` directory,
//! into which third-party packages are installed.
//!
//! The routines exposed by this module have different behaviour depending
//! on the platform of the *host machine*, which may be
//! different from the *target platform for type checking*. (A user
//! might be running red-knot on a Windows machine, but might
//! reasonably ask us to type-check code assuming that the code runs
//! on Linux.)
use std::fmt;
use std::io;
use std::num::NonZeroUsize;
use std::ops::Deref;
use ruff_db::system::{System, SystemPath, SystemPathBuf};
use crate::PythonVersion;
type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
/// Abstraction for a Python virtual environment.
///
/// Most of this information is derived from the virtual environment's `pyvenv.cfg` file.
/// The format of this file is not defined anywhere, and exactly which keys are present
/// depends on the tool that was used to create the virtual environment.
#[derive(Debug)]
pub(crate) struct VirtualEnvironment {
venv_path: SysPrefixPath,
base_executable_home_path: PythonHomePath,
include_system_site_packages: bool,
/// The version of the Python executable that was used to create this virtual environment.
///
/// The Python version is encoded under different keys and in different formats
/// by different virtual-environment creation tools,
/// and the key is never read by the standard-library `site.py` module,
/// so it's possible that we might not be able to find this information
/// in an acceptable format under any of the keys we expect.
/// This field will be `None` if so.
version: Option<PythonVersion>,
}
impl VirtualEnvironment {
pub(crate) fn new(
path: impl AsRef<SystemPath>,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> {
Self::new_impl(path.as_ref(), system)
}
fn new_impl(path: &SystemPath, system: &dyn System) -> SitePackagesDiscoveryResult<Self> {
fn pyvenv_cfg_line_number(index: usize) -> NonZeroUsize {
index.checked_add(1).and_then(NonZeroUsize::new).unwrap()
}
let venv_path = SysPrefixPath::new(path, system)?;
let pyvenv_cfg_path = venv_path.join("pyvenv.cfg");
tracing::debug!("Attempting to parse virtual environment metadata at '{pyvenv_cfg_path}'");
let pyvenv_cfg = system
.read_to_string(&pyvenv_cfg_path)
.map_err(SitePackagesDiscoveryError::NoPyvenvCfgFile)?;
let mut include_system_site_packages = false;
let mut base_executable_home_path = None;
let mut version_info_string = None;
// A `pyvenv.cfg` file *looks* like a `.ini` file, but actually isn't valid `.ini` syntax!
// The Python standard-library's `site` module parses these files by splitting each line on
// '=' characters, so that's what we should do as well.
//
// See also: https://snarky.ca/how-virtual-environments-work/
for (index, line) in pyvenv_cfg.lines().enumerate() {
if let Some((key, value)) = line.split_once('=') {
let key = key.trim();
if key.is_empty() {
return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
pyvenv_cfg_path,
PyvenvCfgParseErrorKind::MalformedKeyValuePair {
line_number: pyvenv_cfg_line_number(index),
},
));
}
let value = value.trim();
if value.is_empty() {
return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
pyvenv_cfg_path,
PyvenvCfgParseErrorKind::MalformedKeyValuePair {
line_number: pyvenv_cfg_line_number(index),
},
));
}
if value.contains('=') {
return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
pyvenv_cfg_path,
PyvenvCfgParseErrorKind::TooManyEquals {
line_number: pyvenv_cfg_line_number(index),
},
));
}
match key {
"include-system-site-packages" => {
include_system_site_packages = value.eq_ignore_ascii_case("true");
}
"home" => base_executable_home_path = Some(value),
// `virtualenv` and `uv` call this key `version_info`,
// but the stdlib venv module calls it `version`
"version" | "version_info" => version_info_string = Some(value),
_ => continue,
}
}
}
// The `home` key is read by the standard library's `site.py` module,
// so if it's missing from the `pyvenv.cfg` file
// (or the provided value is invalid),
// it's reasonable to consider the virtual environment irredeemably broken.
let Some(base_executable_home_path) = base_executable_home_path else {
return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
pyvenv_cfg_path,
PyvenvCfgParseErrorKind::NoHomeKey,
));
};
let base_executable_home_path = PythonHomePath::new(base_executable_home_path, system)
.map_err(|io_err| {
SitePackagesDiscoveryError::PyvenvCfgParseError(
pyvenv_cfg_path,
PyvenvCfgParseErrorKind::InvalidHomeValue(io_err),
)
})?;
// but the `version`/`version_info` key is not read by the standard library,
// and is provided under different keys depending on which virtual-environment creation tool
// created the `pyvenv.cfg` file. Lenient parsing is appropriate here:
// the file isn't really *invalid* if it doesn't have this key,
// or if the value doesn't parse according to our expectations.
let version = version_info_string.and_then(|version_string| {
let mut version_info_parts = version_string.split('.');
let (major, minor) = (version_info_parts.next()?, version_info_parts.next()?);
PythonVersion::try_from((major, minor)).ok()
});
let metadata = Self {
venv_path,
base_executable_home_path,
include_system_site_packages,
version,
};
tracing::trace!("Resolved metadata for virtual environment: {metadata:?}");
Ok(metadata)
}
/// Return a list of `site-packages` directories that are available from this virtual environment
///
/// See the documentation for `site_packages_dir_from_sys_prefix` for more details.
pub(crate) fn site_packages_directories(
&self,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
let VirtualEnvironment {
venv_path,
base_executable_home_path,
include_system_site_packages,
version,
} = self;
let mut site_packages_directories = vec![site_packages_directory_from_sys_prefix(
venv_path, *version, system,
)?];
if *include_system_site_packages {
let system_sys_prefix =
SysPrefixPath::from_executable_home_path(base_executable_home_path);
// If we fail to resolve the `sys.prefix` path from the base executable home path,
// or if we fail to resolve the `site-packages` from the `sys.prefix` path,
// we should probably print a warning but *not* abort type checking
if let Some(sys_prefix_path) = system_sys_prefix {
match site_packages_directory_from_sys_prefix(&sys_prefix_path, *version, system) {
Ok(site_packages_directory) => {
site_packages_directories.push(site_packages_directory);
}
Err(error) => tracing::warn!(
"{error}. System site-packages will not be used for module resolution."
),
}
} else {
tracing::warn!(
"Failed to resolve `sys.prefix` of the system Python installation \
from the `home` value in the `pyvenv.cfg` file at '{}'. \
System site-packages will not be used for module resolution.",
venv_path.join("pyvenv.cfg")
);
}
}
tracing::debug!("Resolved site-packages directories for this virtual environment are: {site_packages_directories:?}");
Ok(site_packages_directories)
}
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum SitePackagesDiscoveryError {
#[error("Invalid --venv-path argument: {0} could not be canonicalized")]
VenvDirCanonicalizationError(SystemPathBuf, #[source] io::Error),
#[error("Invalid --venv-path argument: {0} does not point to a directory on disk")]
VenvDirIsNotADirectory(SystemPathBuf),
#[error("--venv-path points to a broken venv with no pyvenv.cfg file")]
NoPyvenvCfgFile(#[source] io::Error),
#[error("Failed to parse the pyvenv.cfg file at {0} because {1}")]
PyvenvCfgParseError(SystemPathBuf, PyvenvCfgParseErrorKind),
#[error("Failed to search the `lib` directory of the Python installation at {1} for `site-packages`")]
CouldNotReadLibDirectory(#[source] io::Error, SysPrefixPath),
#[error("Could not find the `site-packages` directory for the Python installation at {0}")]
NoSitePackagesDirFound(SysPrefixPath),
}
/// The various ways in which parsing a `pyvenv.cfg` file could fail
#[derive(Debug)]
pub(crate) enum PyvenvCfgParseErrorKind {
TooManyEquals { line_number: NonZeroUsize },
MalformedKeyValuePair { line_number: NonZeroUsize },
NoHomeKey,
InvalidHomeValue(io::Error),
}
impl fmt::Display for PyvenvCfgParseErrorKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::TooManyEquals { line_number } => {
write!(f, "line {line_number} has too many '=' characters")
}
Self::MalformedKeyValuePair { line_number } => write!(
f,
"line {line_number} has a malformed `<key> = <value>` pair"
),
Self::NoHomeKey => f.write_str("the file does not have a `home` key"),
Self::InvalidHomeValue(io_err) => {
write!(
f,
"the following error was encountered \
when trying to resolve the `home` value to a directory on disk: {io_err}"
)
}
}
}
}
/// Attempt to retrieve the `site-packages` directory
/// associated with a given Python installation.
///
/// The location of the `site-packages` directory can vary according to the
/// Python version that this installation represents. The Python version may
/// or may not be known at this point, which is why the `python_version`
/// parameter is an `Option`.
fn site_packages_directory_from_sys_prefix(
sys_prefix_path: &SysPrefixPath,
python_version: Option<PythonVersion>,
system: &dyn System,
) -> SitePackagesDiscoveryResult<SystemPathBuf> {
tracing::debug!("Searching for site-packages directory in {sys_prefix_path}");
if cfg!(target_os = "windows") {
let site_packages = sys_prefix_path.join(r"Lib\site-packages");
return system
.is_directory(&site_packages)
.then_some(site_packages)
.ok_or(SitePackagesDiscoveryError::NoSitePackagesDirFound(
sys_prefix_path.to_owned(),
));
}
// In the Python standard library's `site.py` module (used for finding `site-packages`
// at runtime), we can find this in [the non-Windows branch]:
//
// ```py
// libdirs = [sys.platlibdir]
// if sys.platlibdir != "lib":
// libdirs.append("lib")
// ```
//
// Pyright therefore searches for both a `lib/python3.X/site-packages` directory
// and a `lib64/python3.X/site-packages` directory on non-MacOS Unix systems,
// since `sys.platlibdir` can sometimes be equal to `"lib64"`.
//
// However, we only care about the `site-packages` directory insofar as it allows
// us to discover Python source code that can be used for inferring type
// information regarding third-party dependencies. That means that we don't need
// to care about any possible `lib64/site-packages` directories, since
// [the `sys`-module documentation] states that `sys.platlibdir` is *only* ever
// used for C extensions, never for pure-Python modules.
//
// [the non-Windows branch]: https://github.com/python/cpython/blob/a8be8fc6c4682089be45a87bd5ee1f686040116c/Lib/site.py#L401-L410
// [the `sys`-module documentation]: https://docs.python.org/3/library/sys.html#sys.platlibdir
// If we were able to figure out what Python version this installation is,
// we should be able to avoid iterating through all items in the `lib/` directory:
if let Some(version) = python_version {
let expected_path = sys_prefix_path.join(format!("lib/python{version}/site-packages"));
if system.is_directory(&expected_path) {
return Ok(expected_path);
}
if version.free_threaded_build_available() {
// Nearly the same as `expected_path`, but with an additional `t` after {version}:
let alternative_path =
sys_prefix_path.join(format!("lib/python{version}t/site-packages"));
if system.is_directory(&alternative_path) {
return Ok(alternative_path);
}
}
}
// Either we couldn't figure out the version before calling this function
// (e.g., from a `pyvenv.cfg` file if this was a venv),
// or we couldn't find a `site-packages` folder at the expected location given
// the parsed version
//
// Note: the `python3.x` part of the `site-packages` path can't be computed from
// the `--target-version` the user has passed, as they might be running Python 3.12 locally
// even if they've requested that we type check their code "as if" they're running 3.8.
for entry_result in system
.read_directory(&sys_prefix_path.join("lib"))
.map_err(|io_err| {
SitePackagesDiscoveryError::CouldNotReadLibDirectory(io_err, sys_prefix_path.to_owned())
})?
{
let Ok(entry) = entry_result else {
continue;
};
if !entry.file_type().is_directory() {
continue;
}
let mut path = entry.into_path();
let name = path
.file_name()
.expect("File name to be non-null because path is guaranteed to be a child of `lib`");
if !name.starts_with("python3.") {
continue;
}
path.push("site-packages");
if system.is_directory(&path) {
return Ok(path);
}
}
Err(SitePackagesDiscoveryError::NoSitePackagesDirFound(
sys_prefix_path.to_owned(),
))
}
/// A path that represents the value of [`sys.prefix`] at runtime in Python
/// for a given Python executable.
///
/// For the case of a virtual environment, where a
/// Python binary is at `/.venv/bin/python`, `sys.prefix` is the path to
/// the virtual environment the Python binary lies inside, i.e. `/.venv`,
/// and `site-packages` will be at `.venv/lib/python3.X/site-packages`.
/// System Python installations generally work the same way: if a system
/// Python installation lies at `/opt/homebrew/bin/python`, `sys.prefix`
/// will be `/opt/homebrew`, and `site-packages` will be at
/// `/opt/homebrew/lib/python3.X/site-packages`.
///
/// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix
#[derive(Debug, PartialEq, Eq, Clone)]
pub(crate) struct SysPrefixPath(SystemPathBuf);
impl SysPrefixPath {
fn new(
unvalidated_path: impl AsRef<SystemPath>,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> {
Self::new_impl(unvalidated_path.as_ref(), system)
}
fn new_impl(
unvalidated_path: &SystemPath,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> {
// It's important to resolve symlinks here rather than simply making the path absolute,
// since system Python installations often only put symlinks in the "expected"
// locations for `home` and `site-packages`
let canonicalized = system
.canonicalize_path(unvalidated_path)
.map_err(|io_err| {
SitePackagesDiscoveryError::VenvDirCanonicalizationError(
unvalidated_path.to_path_buf(),
io_err,
)
})?;
system
.is_directory(&canonicalized)
.then_some(Self(canonicalized))
.ok_or_else(|| {
SitePackagesDiscoveryError::VenvDirIsNotADirectory(unvalidated_path.to_path_buf())
})
}
fn from_executable_home_path(path: &PythonHomePath) -> Option<Self> {
// No need to check whether `path.parent()` is a directory:
// the parent of a canonicalised path that is known to exist
// is guaranteed to be a directory.
if cfg!(target_os = "windows") {
Some(Self(path.to_path_buf()))
} else {
path.parent().map(|path| Self(path.to_path_buf()))
}
}
}
impl Deref for SysPrefixPath {
type Target = SystemPath;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl fmt::Display for SysPrefixPath {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "`sys.prefix` path '{}'", self.0)
}
}
/// The value given by the `home` key in `pyvenv.cfg` files.
///
/// This is equivalent to `{sys_prefix_path}/bin`, and points
/// to a directory in which a Python executable can be found.
/// Confusingly, it is *not* the same as the [`PYTHONHOME`]
/// environment variable that Python provides! However, it's
/// consistent among all mainstream creators of Python virtual
/// environments (the stdlib Python `venv` module, the third-party
/// `virtualenv` library, and `uv`), was specified by
/// [the original PEP adding the `venv` module],
/// and it's one of the few fields that's read by the Python
/// standard library's `site.py` module.
///
/// Although it doesn't appear to be specified anywhere,
/// all existing virtual environment tools always use an absolute path
/// for the `home` value, and the Python standard library also assumes
/// that the `home` value will be an absolute path.
///
/// Other values, such as the path to the Python executable or the
/// base-executable `sys.prefix` value, are either only provided in
/// `pyvenv.cfg` files by some virtual-environment creators,
/// or are included under different keys depending on which
/// virtual-environment creation tool you've used.
///
/// [`PYTHONHOME`]: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
/// [the original PEP adding the `venv` module]: https://peps.python.org/pep-0405/
#[derive(Debug, PartialEq, Eq)]
struct PythonHomePath(SystemPathBuf);
impl PythonHomePath {
fn new(path: impl AsRef<SystemPath>, system: &dyn System) -> io::Result<Self> {
let path = path.as_ref();
// It's important to resolve symlinks here rather than simply making the path absolute,
// since system Python installations often only put symlinks in the "expected"
// locations for `home` and `site-packages`
let canonicalized = system.canonicalize_path(path)?;
system
.is_directory(&canonicalized)
.then_some(Self(canonicalized))
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "not a directory"))
}
}
impl Deref for PythonHomePath {
type Target = SystemPath;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl fmt::Display for PythonHomePath {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "`home` location '{}'", self.0)
}
}
impl PartialEq<SystemPath> for PythonHomePath {
fn eq(&self, other: &SystemPath) -> bool {
&*self.0 == other
}
}
impl PartialEq<SystemPathBuf> for PythonHomePath {
fn eq(&self, other: &SystemPathBuf) -> bool {
self == &**other
}
}
#[cfg(test)]
mod tests {
use ruff_db::system::TestSystem;
use super::*;
struct VirtualEnvironmentTester {
system: TestSystem,
minor_version: u8,
free_threaded: bool,
system_site_packages: bool,
pyvenv_cfg_version_field: Option<&'static str>,
}
impl VirtualEnvironmentTester {
/// Builds a mock virtual environment, and returns the path to the venv
fn build_mock_venv(&self) -> SystemPathBuf {
let VirtualEnvironmentTester {
system,
minor_version,
system_site_packages,
free_threaded,
pyvenv_cfg_version_field,
} = self;
let memory_fs = system.memory_file_system();
let unix_site_packages = if *free_threaded {
format!("lib/python3.{minor_version}t/site-packages")
} else {
format!("lib/python3.{minor_version}/site-packages")
};
let system_install_sys_prefix =
SystemPathBuf::from(&*format!("/Python3.{minor_version}"));
let (system_home_path, system_exe_path, system_site_packages_path) =
if cfg!(target_os = "windows") {
let system_home_path = system_install_sys_prefix.clone();
let system_exe_path = system_home_path.join("python.exe");
let system_site_packages_path =
system_install_sys_prefix.join(r"Lib\site-packages");
(system_home_path, system_exe_path, system_site_packages_path)
} else {
let system_home_path = system_install_sys_prefix.join("bin");
let system_exe_path = system_home_path.join("python");
let system_site_packages_path =
system_install_sys_prefix.join(&unix_site_packages);
(system_home_path, system_exe_path, system_site_packages_path)
};
memory_fs.write_file(system_exe_path, "").unwrap();
memory_fs
.create_directory_all(&system_site_packages_path)
.unwrap();
let venv_sys_prefix = SystemPathBuf::from("/.venv");
let (venv_exe, site_packages_path) = if cfg!(target_os = "windows") {
(
venv_sys_prefix.join(r"Scripts\python.exe"),
venv_sys_prefix.join(r"Lib\site-packages"),
)
} else {
(
venv_sys_prefix.join("bin/python"),
venv_sys_prefix.join(&unix_site_packages),
)
};
memory_fs.write_file(&venv_exe, "").unwrap();
memory_fs.create_directory_all(&site_packages_path).unwrap();
let pyvenv_cfg_path = venv_sys_prefix.join("pyvenv.cfg");
let mut pyvenv_cfg_contents = format!("home = {system_home_path}\n");
if let Some(version_field) = pyvenv_cfg_version_field {
pyvenv_cfg_contents.push_str(version_field);
pyvenv_cfg_contents.push('\n');
}
// Deliberately using weird casing here to test that our pyvenv.cfg parsing is case-insensitive:
if *system_site_packages {
pyvenv_cfg_contents.push_str("include-system-site-packages = TRuE\n");
}
memory_fs
.write_file(pyvenv_cfg_path, &pyvenv_cfg_contents)
.unwrap();
venv_sys_prefix
}
fn test(self) {
let venv_path = self.build_mock_venv();
let venv = VirtualEnvironment::new(venv_path.clone(), &self.system).unwrap();
assert_eq!(
venv.venv_path,
SysPrefixPath(self.system.canonicalize_path(&venv_path).unwrap())
);
assert_eq!(venv.include_system_site_packages, self.system_site_packages);
if self.pyvenv_cfg_version_field.is_some() {
assert_eq!(
venv.version,
Some(PythonVersion {
major: 3,
minor: self.minor_version
})
);
} else {
assert_eq!(venv.version, None);
}
let expected_home = if cfg!(target_os = "windows") {
SystemPathBuf::from(&*format!(r"\Python3.{}", self.minor_version))
} else {
SystemPathBuf::from(&*format!("/Python3.{}/bin", self.minor_version))
};
assert_eq!(venv.base_executable_home_path, expected_home);
let site_packages_directories = venv.site_packages_directories(&self.system).unwrap();
let expected_venv_site_packages = if cfg!(target_os = "windows") {
SystemPathBuf::from(r"\.venv\Lib\site-packages")
} else if self.free_threaded {
SystemPathBuf::from(&*format!(
"/.venv/lib/python3.{}t/site-packages",
self.minor_version
))
} else {
SystemPathBuf::from(&*format!(
"/.venv/lib/python3.{}/site-packages",
self.minor_version
))
};
let expected_system_site_packages = if cfg!(target_os = "windows") {
SystemPathBuf::from(&*format!(
r"\Python3.{}\Lib\site-packages",
self.minor_version
))
} else if self.free_threaded {
SystemPathBuf::from(&*format!(
"/Python3.{minor_version}/lib/python3.{minor_version}t/site-packages",
minor_version = self.minor_version
))
} else {
SystemPathBuf::from(&*format!(
"/Python3.{minor_version}/lib/python3.{minor_version}/site-packages",
minor_version = self.minor_version
))
};
if self.system_site_packages {
assert_eq!(
&site_packages_directories,
&[expected_venv_site_packages, expected_system_site_packages]
);
} else {
assert_eq!(&site_packages_directories, &[expected_venv_site_packages]);
}
}
}
#[test]
fn can_find_site_packages_directory_no_version_field_in_pyvenv_cfg() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 12,
free_threaded: false,
system_site_packages: false,
pyvenv_cfg_version_field: None,
};
tester.test();
}
#[test]
fn can_find_site_packages_directory_venv_style_version_field_in_pyvenv_cfg() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 12,
free_threaded: false,
system_site_packages: false,
pyvenv_cfg_version_field: Some("version = 3.12"),
};
tester.test();
}
#[test]
fn can_find_site_packages_directory_uv_style_version_field_in_pyvenv_cfg() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 12,
free_threaded: false,
system_site_packages: false,
pyvenv_cfg_version_field: Some("version_info = 3.12"),
};
tester.test();
}
#[test]
fn can_find_site_packages_directory_virtualenv_style_version_field_in_pyvenv_cfg() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 12,
free_threaded: false,
system_site_packages: false,
pyvenv_cfg_version_field: Some("version_info = 3.12.0rc2"),
};
tester.test();
}
#[test]
fn can_find_site_packages_directory_freethreaded_build() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 13,
free_threaded: true,
system_site_packages: false,
pyvenv_cfg_version_field: Some("version_info = 3.13"),
};
tester.test();
}
#[test]
fn finds_system_site_packages() {
let tester = VirtualEnvironmentTester {
system: TestSystem::default(),
minor_version: 13,
free_threaded: true,
system_site_packages: true,
pyvenv_cfg_version_field: Some("version_info = 3.13"),
};
tester.test();
}
#[test]
fn reject_venv_that_does_not_exist() {
let system = TestSystem::default();
assert!(matches!(
VirtualEnvironment::new("/.venv", &system),
Err(SitePackagesDiscoveryError::VenvDirIsNotADirectory(_))
));
}
#[test]
fn reject_venv_with_no_pyvenv_cfg_file() {
let system = TestSystem::default();
system
.memory_file_system()
.create_directory_all("/.venv")
.unwrap();
assert!(matches!(
VirtualEnvironment::new("/.venv", &system),
Err(SitePackagesDiscoveryError::NoPyvenvCfgFile(_))
));
}
#[test]
fn parsing_pyvenv_cfg_with_too_many_equals() {
let system = TestSystem::default();
let memory_fs = system.memory_file_system();
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs
.write_file(&pyvenv_cfg_path, "home = bar = /.venv/bin")
.unwrap();
let venv_result = VirtualEnvironment::new("/.venv", &system);
assert!(matches!(
venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
path,
PyvenvCfgParseErrorKind::TooManyEquals { line_number }
))
if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
));
}
#[test]
fn parsing_pyvenv_cfg_with_key_but_no_value_fails() {
let system = TestSystem::default();
let memory_fs = system.memory_file_system();
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs.write_file(&pyvenv_cfg_path, "home =").unwrap();
let venv_result = VirtualEnvironment::new("/.venv", &system);
assert!(matches!(
venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
path,
PyvenvCfgParseErrorKind::MalformedKeyValuePair { line_number }
))
if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
));
}
#[test]
fn parsing_pyvenv_cfg_with_value_but_no_key_fails() {
let system = TestSystem::default();
let memory_fs = system.memory_file_system();
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs
.write_file(&pyvenv_cfg_path, "= whatever")
.unwrap();
let venv_result = VirtualEnvironment::new("/.venv", &system);
assert!(matches!(
venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
path,
PyvenvCfgParseErrorKind::MalformedKeyValuePair { line_number }
))
if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
));
}
#[test]
fn parsing_pyvenv_cfg_with_no_home_key_fails() {
let system = TestSystem::default();
let memory_fs = system.memory_file_system();
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs.write_file(&pyvenv_cfg_path, "").unwrap();
let venv_result = VirtualEnvironment::new("/.venv", &system);
assert!(matches!(
venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
path,
PyvenvCfgParseErrorKind::NoHomeKey
))
if path == pyvenv_cfg_path
));
}
#[test]
fn parsing_pyvenv_cfg_with_invalid_home_key_fails() {
let system = TestSystem::default();
let memory_fs = system.memory_file_system();
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs
.write_file(&pyvenv_cfg_path, "home = foo")
.unwrap();
let venv_result = VirtualEnvironment::new("/.venv", &system);
assert!(matches!(
venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
path,
PyvenvCfgParseErrorKind::InvalidHomeValue(_)
))
if path == pyvenv_cfg_path
));
}
}

9
crates/red_knot_python_semantic/src/types.rs
View file

@ -392,14 +392,9 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::default(),
search_paths: SearchPathSettings {
extra_paths: Vec::new(),
src_root: SystemPathBuf::from("/src"),
site_packages: vec![],
custom_typeshed: None,
},
search_paths: SearchPathSettings::new(SystemPathBuf::from("/src")),
},
)
.expect("Valid search path settings");

15
crates/red_knot_python_semantic/src/types/infer.rs
View file

@ -1866,14 +1866,9 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::default(),
search_paths: SearchPathSettings {
extra_paths: Vec::new(),
src_root,
site_packages: vec![],
custom_typeshed: None,
},
search_paths: SearchPathSettings::new(src_root),
},
)
.expect("Valid search path settings");
@ -1893,13 +1888,11 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
&ProgramSettings {
target_version: PythonVersion::default(),
search_paths: SearchPathSettings {
extra_paths: Vec::new(),
src_root,
site_packages: vec![],
custom_typeshed: Some(SystemPathBuf::from(typeshed)),
..SearchPathSettings::new(src_root)
},
},
)