mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-15 16:10:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

[ty] Move search path resolution to Options::to_program_settings (#18937)

Browse source
This commit is contained in:
Micha Reiser 2025-06-25 18:00:38 +02:00 • committed by GitHub
parent 8b22992988
commit 5d546c600a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 481 additions and 549 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ty_python_semantic/src/module_resolver/mod.rs
View file

@ -1,8 +1,10 @@
use std::iter::FusedIterator;
pub use module::{KnownModule, Module};
pub use path::SearchPathValidationError;
pub use resolver::SearchPaths;
pub(crate) use resolver::file_to_module;
pub use resolver::resolve_module;
pub(crate) use resolver::{SearchPaths, file_to_module};
use ruff_db::system::SystemPath;
use crate::Db;

66
crates/ty_python_semantic/src/module_resolver/path.rs
View file

@ -9,7 +9,6 @@ use ruff_db::system::{System, SystemPath, SystemPathBuf};
use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult, typeshed_versions};
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::module_resolver::resolver::ResolverContext;
use crate::site_packages::SitePackagesDiscoveryError;
@ -325,62 +324,37 @@ fn query_stdlib_version(
/// If validation fails for a search path derived from the user settings,
/// a message must be displayed to the user,
/// as type checking cannot be done reliably in these circumstances.
#[derive(Debug)]
pub(crate) enum SearchPathValidationError {
#[derive(Debug, thiserror::Error)]
pub enum SearchPathValidationError {
/// The path provided by the user was not a directory
#[error("{0} does not point to a directory")]
NotADirectory(SystemPathBuf),
/// The path provided by the user is a directory,
/// but no `stdlib/` subdirectory exists.
/// (This is only relevant for stdlib search paths.)
#[error("The directory at {0} has no `stdlib/` subdirectory")]
NoStdlibSubdirectory(SystemPathBuf),
/// The typeshed path provided by the user is a directory,
/// but `stdlib/VERSIONS` could not be read.
/// (This is only relevant for stdlib search paths.)
#[error("Failed to read the custom typeshed versions file '{path}'")]
FailedToReadVersionsFile {
path: SystemPathBuf,
#[source]
error: std::io::Error,
},
/// The path provided by the user is a directory,
/// and a `stdlib/VERSIONS` file exists, but it fails to parse.
/// (This is only relevant for stdlib search paths.)
#[error(transparent)]
VersionsParseError(TypeshedVersionsParseError),
/// Failed to discover the site-packages for the configured virtual environment.
SitePackagesDiscovery(SitePackagesDiscoveryError),
}
impl fmt::Display for SearchPathValidationError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotADirectory(path) => write!(f, "{path} does not point to a directory"),
Self::NoStdlibSubdirectory(path) => {
write!(f, "The directory at {path} has no `stdlib/` subdirectory")
}
Self::FailedToReadVersionsFile { path, error } => {
write!(
f,
"Failed to read the custom typeshed versions file '{path}': {error}"
)
}
Self::VersionsParseError(underlying_error) => underlying_error.fmt(f),
SearchPathValidationError::SitePackagesDiscovery(error) => {
write!(f, "Failed to discover the site-packages directory: {error}")
}
}
}
}
impl std::error::Error for SearchPathValidationError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
if let Self::VersionsParseError(underlying_error) = self {
Some(underlying_error)
} else {
None
}
}
#[error("Failed to discover the site-packages directory")]
SitePackagesDiscovery(#[source] SitePackagesDiscoveryError),
}
impl From<TypeshedVersionsParseError> for SearchPathValidationError {
@ -459,8 +433,10 @@ impl SearchPath {
}
/// Create a new standard-library search path pointing to a custom directory on disk
pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: &SystemPath) -> SearchPathResult<Self> {
let system = db.system();
pub(crate) fn custom_stdlib(
system: &dyn System,
typeshed: &SystemPath,
) -> SearchPathResult<Self> {
if !system.is_directory(typeshed) {
return Err(SearchPathValidationError::NotADirectory(
typeshed.to_path_buf(),
@ -528,6 +504,10 @@ impl SearchPath {
)
}
pub(crate) fn is_first_party(&self) -> bool {
matches!(&*self.0, SearchPathInner::FirstParty(_))
}
fn is_valid_extension(&self, extension: &str) -> bool {
if self.is_standard_library() {
extension == "pyi"
@ -707,7 +687,7 @@ mod tests {
.build();
assert_eq!(
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.with_py_extension(),
@ -715,7 +695,7 @@ mod tests {
);
assert_eq!(
&SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
&SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap())
.unwrap()
.join("foo")
.with_pyi_extension(),
@ -826,7 +806,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_mocked_typeshed(MockedTypeshed::default())
.build();
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.push("bar.py");
@ -838,7 +818,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_mocked_typeshed(MockedTypeshed::default())
.build();
SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap())
.unwrap()
.to_module_path()
.push("bar.rs");
@ -870,7 +850,7 @@ mod tests {
.with_mocked_typeshed(MockedTypeshed::default())
.build();
let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
let root = SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()).unwrap();
// Must have a `.pyi` extension or no extension:
let bad_absolute_path = SystemPath::new("foo/stdlib/x.py");
@ -918,7 +898,7 @@ mod tests {
.with_mocked_typeshed(typeshed)
.with_python_version(python_version)
.build();
let stdlib = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
let stdlib = SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()).unwrap();
(db, stdlib)
}

83
crates/ty_python_semantic/src/module_resolver/resolver.rs
View file

@ -142,9 +142,9 @@ pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator {
Program::get(db).search_paths(db).iter(db)
}
#[derive(Debug, PartialEq, Eq)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SearchPaths {
/// Search paths that have been statically determined purely from reading Ruff's configuration settings.
/// Search paths that have been statically determined purely from reading ty's configuration settings.
/// These shouldn't ever change unless the config settings themselves change.
static_paths: Vec<SearchPath>,
@ -174,8 +174,9 @@ impl SearchPaths {
///
/// [module resolution order]: https://typing.python.org/en/latest/spec/distributing.html#import-resolution-ordering
pub(crate) fn from_settings(
db: &dyn Db,
settings: &SearchPathSettings,
system: &dyn System,
vendored: &VendoredFileSystem,
) -> Result<Self, SearchPathValidationError> {
fn canonicalize(path: &SystemPath, system: &dyn System) -> SystemPathBuf {
system
@ -190,14 +191,10 @@ impl SearchPaths {
python_path,
} = settings;
let system = db.system();
let files = db.files();
let mut static_paths = vec![];
for path in extra_paths {
let path = canonicalize(path, system);
files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
tracing::debug!("Adding extra search-path '{path}'");
static_paths.push(SearchPath::extra(system, path)?);
@ -212,8 +209,6 @@ impl SearchPaths {
let typeshed = canonicalize(typeshed, system);
tracing::debug!("Adding custom-stdlib search path '{typeshed}'");
files.try_add_root(db.upcast(), &typeshed, FileRootKind::LibrarySearchPath);
let versions_path = typeshed.join("stdlib/VERSIONS");
let versions_content = system.read_to_string(&versions_path).map_err(|error| {
@ -225,13 +220,13 @@ impl SearchPaths {
let parsed: TypeshedVersions = versions_content.parse()?;
let search_path = SearchPath::custom_stdlib(db, &typeshed)?;
let search_path = SearchPath::custom_stdlib(system, &typeshed)?;
(parsed, search_path)
} else {
tracing::debug!("Using vendored stdlib");
(
vendored_typeshed_versions(db),
vendored_typeshed_versions(vendored),
SearchPath::vendored_stdlib(),
)
};
@ -282,7 +277,6 @@ impl SearchPaths {
for path in site_packages_paths {
tracing::debug!("Adding site-packages search path '{path}'");
files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
site_packages.push(SearchPath::site_packages(system, path)?);
}
@ -313,6 +307,17 @@ impl SearchPaths {
})
}
pub(crate) fn try_register_static_roots(&self, db: &dyn Db) {
let files = db.files();
for path in self.static_paths.iter().chain(self.site_packages.iter()) {
if let Some(system_path) = path.as_system_path() {
if !path.is_first_party() {
files.try_add_root(db.upcast(), system_path, FileRootKind::LibrarySearchPath);
}
}
}
}
pub(super) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
SearchPathIterator {
db,
@ -1482,20 +1487,20 @@ mod tests {
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource {
python_version: PythonVersionWithSource {
version: PythonVersion::PY38,
source: PythonVersionSource::default(),
}),
},
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_roots: vec![src.clone()],
custom_typeshed: Some(custom_typeshed),
python_path: PythonPath::KnownSitePackages(vec![site_packages]),
},
..SearchPathSettings::new(vec![src.clone()])
}
.to_search_paths(db.system(), db.vendored())
.expect("Valid search path settings"),
},
)
.context("Invalid program settings")?;
);
let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap();
let bar_module = resolve_module(&db, &ModuleName::new_static("bar").unwrap()).unwrap();
@ -2001,20 +2006,19 @@ not_a_directory
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource::default()),
python_version: PythonVersionWithSource::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_roots: vec![SystemPathBuf::from("/src")],
custom_typeshed: None,
python_path: PythonPath::KnownSitePackages(vec![
venv_site_packages,
system_site_packages,
]),
},
..SearchPathSettings::new(vec![SystemPathBuf::from("/src")])
}
.to_search_paths(db.system(), db.vendored())
.expect("Valid search path settings"),
},
)
.expect("Valid program settings");
);
// The editable installs discovered from the `.pth` file in the first `site-packages` directory
// take precedence over the second `site-packages` directory...
@ -2080,17 +2084,13 @@ not_a_directory
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource::default()),
python_version: PythonVersionWithSource::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_roots: vec![src],
custom_typeshed: None,
python_path: PythonPath::KnownSitePackages(vec![]),
},
search_paths: SearchPathSettings::new(vec![src])
.to_search_paths(db.system(), db.vendored())
.expect("valid search path settings"),
},
)
.expect("Valid program settings");
);
// Now try to resolve the module `A` (note the capital `A` instead of `a`).
let a_module_name = ModuleName::new_static("A").unwrap();
@ -2123,17 +2123,16 @@ not_a_directory
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource::default()),
python_version: PythonVersionWithSource::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings {
extra_paths: vec![],
src_roots: vec![project_directory],
custom_typeshed: None,
python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
},
..SearchPathSettings::new(vec![project_directory])
}
.to_search_paths(db.system(), db.vendored())
.unwrap(),
},
)
.unwrap();
);
let foo_module_file = File::new(&db, FilePath::System(installed_foo_module));
let module = file_to_module(&db, foo_module_file).unwrap();

26
crates/ty_python_semantic/src/module_resolver/testing.rs
View file

@ -1,3 +1,4 @@
use ruff_db::Db;
use ruff_db::system::{
DbWithTestSystem as _, DbWithWritableSystem as _, SystemPath, SystemPathBuf,
};
@ -237,20 +238,20 @@ impl TestCaseBuilder<MockedTypeshed> {
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource {
python_version: PythonVersionWithSource {
version: python_version,
source: PythonVersionSource::default(),
}),
},
python_platform,
search_paths: SearchPathSettings {
extra_paths: vec![],
src_roots: vec![src.clone()],
custom_typeshed: Some(typeshed.clone()),
python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
},
..SearchPathSettings::new(vec![src.clone()])
}
.to_search_paths(db.system(), db.vendored())
.expect("valid search path settings"),
},
)
.expect("Valid program settings");
);
TestCase {
db,
@ -298,18 +299,19 @@ impl TestCaseBuilder<VendoredTypeshed> {
Program::from_settings(
&db,
ProgramSettings {
python_version: Some(PythonVersionWithSource {
python_version: PythonVersionWithSource {
version: python_version,
source: PythonVersionSource::default(),
}),
},
python_platform,
search_paths: SearchPathSettings {
python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
..SearchPathSettings::new(vec![src.clone()])
},
}
.to_search_paths(db.system(), db.vendored())
.expect("valid search path settings"),
},
)
.expect("Valid search path settings");
);
TestCase {
db,

23
crates/ty_python_semantic/src/module_resolver/typeshed.rs
View file

@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroUsize};
use std::ops::{RangeFrom, RangeInclusive};
use std::str::FromStr;
use ruff_db::vendored::VendoredFileSystem;
use ruff_python_ast::{PythonVersion, PythonVersionDeserializationError};
use rustc_hash::FxHashMap;
@ -11,9 +12,11 @@ use crate::Program;
use crate::db::Db;
use crate::module_name::ModuleName;
pub(in crate::module_resolver) fn vendored_typeshed_versions(db: &dyn Db) -> TypeshedVersions {
pub(in crate::module_resolver) fn vendored_typeshed_versions(
vendored: &VendoredFileSystem,
) -> TypeshedVersions {
TypeshedVersions::from_str(
&db.vendored()
&vendored
.read_to_string("stdlib/VERSIONS")
.expect("The vendored typeshed stubs should contain a VERSIONS file"),
)
@ -25,7 +28,7 @@ pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions {
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub(crate) struct TypeshedVersionsParseError {
pub struct TypeshedVersionsParseError {
line_number: Option<NonZeroU16>,
reason: TypeshedVersionsParseErrorKind,
}
@ -71,7 +74,7 @@ pub(crate) enum TypeshedVersionsParseErrorKind {
VersionParseError(#[from] PythonVersionDeserializationError),
}
#[derive(Debug, PartialEq, Eq)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct TypeshedVersions(FxHashMap<ModuleName, PyVersionRange>);
impl TypeshedVersions {
@ -305,11 +308,8 @@ mod tests {
use std::num::{IntErrorKind, NonZeroU16};
use std::path::Path;
use insta::assert_snapshot;
use crate::db::tests::TestDb;
use super::*;
use insta::assert_snapshot;
const TYPESHED_STDLIB_DIR: &str = "stdlib";
@ -329,9 +329,7 @@ mod tests {
#[test]
fn can_parse_vendored_versions_file() {
let db = TestDb::new();
let versions = vendored_typeshed_versions(&db);
let versions = vendored_typeshed_versions(ty_vendored::file_system());
assert!(versions.len() > 100);
assert!(versions.len() < 1000);
@ -368,8 +366,7 @@ mod tests {
#[test]
fn typeshed_versions_consistent_with_vendored_stubs() {
let db = TestDb::new();
let vendored_typeshed_versions = vendored_typeshed_versions(&db);
let vendored_typeshed_versions = vendored_typeshed_versions(ty_vendored::file_system());
let vendored_typeshed_dir =
Path::new(env!("CARGO_MANIFEST_DIR")).join("../ty_vendored/vendor/typeshed");