diff --git a/crates/ruff/tests/analyze_graph.rs b/crates/ruff/tests/analyze_graph.rs index b6635d96bf..acfacac190 100644 --- a/crates/ruff/tests/analyze_graph.rs +++ b/crates/ruff/tests/analyze_graph.rs @@ -566,7 +566,9 @@ fn venv() -> Result<()> { ----- stderr ----- ruff failed Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Invalid `--python` argument `none`: does not point to a Python executable or a directory on disk + Cause: Failed to discover the site-packages directory + Cause: Invalid `--python` argument `none`: does not point to a Python executable or a directory on disk + Cause: No such file or directory (os error 2) "); }); diff --git a/crates/ruff_graph/src/db.rs b/crates/ruff_graph/src/db.rs index b7056ece58..cae2b56e83 100644 --- a/crates/ruff_graph/src/db.rs +++ b/crates/ruff_graph/src/db.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use std::sync::Arc; use zip::CompressionMethod; @@ -42,17 +42,21 @@ impl ModuleDb { } let db = Self::default(); + let search_paths = search_paths + .to_search_paths(db.system(), db.vendored()) + .context("Invalid search path settings")?; + Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: python_version, source: PythonVersionSource::default(), - }), + }, python_platform: PythonPlatform::default(), search_paths, }, - )?; + ); Ok(db) } diff --git a/crates/ty/tests/cli/main.rs b/crates/ty/tests/cli/main.rs index de98e33660..ec6e83c2ff 100644 --- a/crates/ty/tests/cli/main.rs +++ b/crates/ty/tests/cli/main.rs @@ -621,6 +621,10 @@ impl CliTest { let mut settings = insta::Settings::clone_current(); settings.add_filter(&tempdir_filter(&project_dir), "/"); settings.add_filter(r#"\\(\w\w|\s|\.|")"#, "/$1"); + settings.add_filter( + r#"The system cannot find the file specified."#, + "No such file or directory", + ); let settings_scope = settings.bind_to_scope(); diff --git a/crates/ty/tests/cli/python_environment.rs b/crates/ty/tests/cli/python_environment.rs index 2e95d060a6..eab7ba743b 100644 --- a/crates/ty/tests/cli/python_environment.rs +++ b/crates/ty/tests/cli/python_environment.rs @@ -590,8 +590,8 @@ fn python_cli_argument_virtual_environment() -> anyhow::Result<()> { ----- stderr ----- WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. ty failed - Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Invalid `--python` argument `/my-venv/foo/some_other_file.txt`: does not point to a Python executable or a directory on disk + Cause: Failed to discover the site-packages directory + Cause: Invalid `--python` argument `/my-venv/foo/some_other_file.txt`: does not point to a Python executable or a directory on disk "); // And so are paths that do not exist on disk @@ -603,8 +603,9 @@ fn python_cli_argument_virtual_environment() -> anyhow::Result<()> { ----- stderr ----- WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. ty failed - Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Invalid `--python` argument `/not-a-directory-or-executable`: does not point to a Python executable or a directory on disk + Cause: Failed to discover the site-packages directory + Cause: Invalid `--python` argument `/not-a-directory-or-executable`: does not point to a Python executable or a directory on disk + Cause: No such file or directory (os error 2) "); Ok(()) @@ -685,8 +686,8 @@ fn config_file_broken_python_setting() -> anyhow::Result<()> { ----- stderr ----- WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. ty failed - Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Invalid `environment.python` setting + Cause: Failed to discover the site-packages directory + Cause: Invalid `environment.python` setting --> Invalid setting in configuration file `/pyproject.toml` | @@ -695,6 +696,8 @@ fn config_file_broken_python_setting() -> anyhow::Result<()> { 11 | python = "not-a-directory-or-executable" | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ does not point to a Python executable or a directory on disk | + + Cause: No such file or directory (os error 2) "#); Ok(()) @@ -722,8 +725,8 @@ fn config_file_python_setting_directory_with_no_site_packages() -> anyhow::Resul ----- stderr ----- WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. ty failed - Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Invalid `environment.python` setting + Cause: Failed to discover the site-packages directory + Cause: Invalid `environment.python` setting --> Invalid setting in configuration file `/pyproject.toml` | @@ -761,8 +764,8 @@ fn unix_system_installation_with_no_lib_directory() -> anyhow::Result<()> { ----- stderr ----- WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. ty failed - Cause: Invalid search path settings - Cause: Failed to discover the site-packages directory: Failed to iterate over the contents of the `lib` directory of the Python installation + Cause: Failed to discover the site-packages directory + Cause: Failed to iterate over the contents of the `lib` directory of the Python installation --> Invalid setting in configuration file `/pyproject.toml` | @@ -771,6 +774,8 @@ fn unix_system_installation_with_no_lib_directory() -> anyhow::Result<()> { 3 | python = "directory-but-no-site-packages" | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | + + Cause: No such file or directory (os error 2) "#); Ok(()) @@ -1049,3 +1054,169 @@ fn environment_root_takes_precedence_over_src_root() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn default_root_src_layout() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("src/foo.py", "foo = 10"), + ("bar.py", "bar = 20"), + ( + "src/main.py", + r#" + from foo import foo + from bar import bar + + print(f"{foo} {bar}") + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. + "); + + Ok(()) +} + +#[test] +fn default_root_project_name_folder() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ( + "pyproject.toml", + r#" + [project] + name = "psycopg" + "#, + ), + ("psycopg/psycopg/foo.py", "foo = 10"), + ("bar.py", "bar = 20"), + ( + "psycopg/psycopg/main.py", + r#" + from psycopg.foo import foo + from bar import bar + + print(f"{foo} {bar}") + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. + "); + + Ok(()) +} + +#[test] +fn default_root_flat_layout() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("app/foo.py", "foo = 10"), + ("bar.py", "bar = 20"), + ( + "app/main.py", + r#" + from app.foo import foo + from bar import bar + + print(f"{foo} {bar}") + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. + "); + + Ok(()) +} + +#[test] +fn default_root_tests_folder() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("src/foo.py", "foo = 10"), + ("tests/bar.py", "bar = 20"), + ( + "tests/test_bar.py", + r#" + from foo import foo + from bar import bar + + print(f"{foo} {bar}") + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. + "); + + Ok(()) +} + +/// If `tests/__init__.py` is present, it is considered a package and `tests` is not added to `sys.path`. +#[test] +fn default_root_tests_package() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("src/foo.py", "foo = 10"), + ("tests/__init__.py", ""), + ("tests/bar.py", "bar = 20"), + ( + "tests/test_bar.py", + r#" + from foo import foo + from bar import bar # expected unresolved import + + print(f"{foo} {bar}") + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @r#" + success: false + exit_code: 1 + ----- stdout ----- + error[unresolved-import]: Cannot resolve imported module `bar` + --> tests/test_bar.py:3:6 + | + 2 | from foo import foo + 3 | from bar import bar # expected unresolved import + | ^^^ + 4 | + 5 | print(f"{foo} {bar}") + | + info: make sure your Python environment is properly configured: https://github.com/astral-sh/ty/blob/main/docs/README.md#python-environment + info: rule `unresolved-import` is enabled by default + + Found 1 diagnostic + + ----- stderr ----- + WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors. + "#); + + Ok(()) +} diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index 71c50ff6eb..da076c6bc7 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -396,16 +396,18 @@ where let mut project = ProjectMetadata::discover(&project_path, &system)?; project.apply_configuration_files(&system)?; - let program_settings = project.to_program_settings(&system); - - for path in program_settings - .search_paths - .extra_paths - .iter() - .chain(program_settings.search_paths.custom_typeshed.as_ref()) - { - std::fs::create_dir_all(path.as_std_path()) - .with_context(|| format!("Failed to create search path `{path}`"))?; + // We need a chance to create the directories here. + if let Some(environment) = project.options().environment.as_ref() { + for path in environment + .extra_paths + .as_deref() + .unwrap_or_default() + .iter() + .chain(environment.typeshed.as_ref()) + { + std::fs::create_dir_all(path.absolute(&project_path, &system).as_std_path()) + .with_context(|| format!("Failed to create search path `{path}`"))?; + } } let mut db = ProjectDatabase::new(project, system)?; diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 9c1c835024..80958f40c6 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -150,6 +150,7 @@ mod tests { use insta::assert_snapshot; use ruff_db::{ + Db as _, files::{File, system_path_to_file}, source::source_text, }; @@ -159,8 +160,7 @@ mod tests { use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ty_python_semantic::{ - Program, ProgramSettings, PythonPath, PythonPlatform, PythonVersionWithSource, - SearchPathSettings, + Program, ProgramSettings, PythonPlatform, PythonVersionWithSource, SearchPathSettings, }; pub(super) fn inlay_hint_test(source: &str) -> InlayHintTest { @@ -188,20 +188,18 @@ mod tests { let file = system_path_to_file(&db, "main.py").expect("newly written file to existing"); + let search_paths = SearchPathSettings::new(vec![SystemPathBuf::from("/")]) + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"); + Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![SystemPathBuf::from("/")], - custom_typeshed: None, - python_path: PythonPath::KnownSitePackages(vec![]), - }, + search_paths, }, - ) - .expect("Default settings to be valid"); + ); InlayHintTest { db, file, range } } diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index f7a1168329..417316a1e4 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -203,14 +203,13 @@ impl HasNavigationTargets for TypeDefinition<'_> { mod tests { use crate::db::tests::TestDb; use insta::internals::SettingsBindDropGuard; - use ruff_db::Upcast; use ruff_db::diagnostic::{Diagnostic, DiagnosticFormat, DisplayDiagnosticConfig}; use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem, SystemPath, SystemPathBuf}; + use ruff_db::{Db, Upcast}; use ruff_text_size::TextSize; use ty_python_semantic::{ - Program, ProgramSettings, PythonPath, PythonPlatform, PythonVersionWithSource, - SearchPathSettings, + Program, ProgramSettings, PythonPlatform, PythonVersionWithSource, SearchPathSettings, }; /// A way to create a simple single-file (named `main.py`) cursor test. @@ -302,20 +301,18 @@ mod tests { cursor = Some(Cursor { file, offset }); } + let search_paths = SearchPathSettings::new(vec![SystemPathBuf::from("/")]) + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"); + Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![SystemPathBuf::from("/")], - custom_typeshed: None, - python_path: PythonPath::KnownSitePackages(vec![]), - }, + search_paths, }, - ) - .expect("Default settings to be valid"); + ); let mut insta_settings = insta::Settings::clone_current(); insta_settings.add_filter(r#"\\(\w\w|\s|\.|")"#, "/$1"); diff --git a/crates/ty_project/src/db.rs b/crates/ty_project/src/db.rs index 654ae540c4..dc9f282109 100644 --- a/crates/ty_project/src/db.rs +++ b/crates/ty_project/src/db.rs @@ -68,8 +68,8 @@ impl ProjectDatabase { // we may want to have a dedicated method for this? // Initialize the `Program` singleton - let program_settings = project_metadata.to_program_settings(db.system()); - Program::from_settings(&db, program_settings)?; + let program_settings = project_metadata.to_program_settings(db.system(), db.vendored())?; + Program::from_settings(&db, program_settings); db.project = Some( Project::from_metadata(&db, project_metadata) diff --git a/crates/ty_project/src/db/changes.rs b/crates/ty_project/src/db/changes.rs index 7a9c989821..f1b203c299 100644 --- a/crates/ty_project/src/db/changes.rs +++ b/crates/ty_project/src/db/changes.rs @@ -228,13 +228,16 @@ impl ProjectDatabase { ); } - let program_settings = metadata.to_program_settings(self.system()); - - let program = Program::get(self); - if let Err(error) = program.update_from_settings(self, program_settings) { - tracing::error!( - "Failed to update the program settings, keeping the old program settings: {error}" - ); + match metadata.to_program_settings(self.system(), self.vendored()) { + Ok(program_settings) => { + let program = Program::get(self); + program.update_from_settings(self, program_settings); + } + Err(error) => { + tracing::error!( + "Failed to convert metadata to program settings, continuing without applying them: {error}" + ); + } } if metadata.root() == project.root(self) { @@ -269,13 +272,16 @@ impl ProjectDatabase { return result; } else if result.custom_stdlib_changed { - let search_paths = project + match project .metadata(self) - .to_program_settings(self.system()) - .search_paths; - - if let Err(error) = program.update_search_paths(self, &search_paths) { - tracing::error!("Failed to set the new search paths: {error}"); + .to_program_settings(self.system(), self.vendored()) + { + Ok(program_settings) => { + program.update_from_settings(self, program_settings); + } + Err(error) => { + tracing::error!("Failed to resolve program settings: {error}"); + } } } diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index f869b06bab..11834036ab 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -714,6 +714,7 @@ mod tests { use crate::Db; use crate::ProjectMetadata; use crate::db::tests::TestDb; + use ruff_db::Db as _; use ruff_db::files::system_path_to_file; use ruff_db::source::source_text; use ruff_db::system::{DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemPathBuf}; @@ -733,12 +734,13 @@ mod tests { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(vec![SystemPathBuf::from(".")]), + search_paths: SearchPathSettings::new(vec![SystemPathBuf::from(".")]) + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"), }, - ) - .expect("Failed to configure program settings"); + ); db.write_file(path, "x = 10")?; let file = system_path_to_file(&db, path).unwrap(); diff --git a/crates/ty_project/src/metadata.rs b/crates/ty_project/src/metadata.rs index 47896e5031..fe99c6df00 100644 --- a/crates/ty_project/src/metadata.rs +++ b/crates/ty_project/src/metadata.rs @@ -1,5 +1,6 @@ use configuration_file::{ConfigurationFile, ConfigurationFileError}; use ruff_db::system::{System, SystemPath, SystemPathBuf}; +use ruff_db::vendored::VendoredFileSystem; use ruff_python_ast::name::Name; use std::sync::Arc; use thiserror::Error; @@ -266,9 +267,13 @@ impl ProjectMetadata { &self.extra_configuration_paths } - pub fn to_program_settings(&self, system: &dyn System) -> ProgramSettings { + pub fn to_program_settings( + &self, + system: &dyn System, + vendored: &VendoredFileSystem, + ) -> anyhow::Result { self.options - .to_program_settings(self.root(), self.name(), system) + .to_program_settings(self.root(), self.name(), system, vendored) } /// Combine the project options with the CLI options where the CLI options take precedence. @@ -977,133 +982,6 @@ expected `.`, `]` Ok(()) } - #[test] - fn no_src_root_src_layout() -> anyhow::Result<()> { - let system = TestSystem::default(); - let root = SystemPathBuf::from("/app"); - - system - .memory_file_system() - .write_file_all( - root.join("src/main.py"), - r#" - print("Hello, world!") - "#, - ) - .context("Failed to write file")?; - - let metadata = ProjectMetadata::discover(&root, &system)?; - let settings = metadata - .options - .to_program_settings(&root, "my_package", &system); - - assert_eq!( - settings.search_paths.src_roots, - vec![root.clone(), root.join("src")] - ); - - Ok(()) - } - - #[test] - fn no_src_root_package_layout() -> anyhow::Result<()> { - let system = TestSystem::default(); - let root = SystemPathBuf::from("/app"); - - system - .memory_file_system() - .write_file_all( - root.join("psycopg/psycopg/main.py"), - r#" - print("Hello, world!") - "#, - ) - .context("Failed to write file")?; - - let metadata = ProjectMetadata::discover(&root, &system)?; - let settings = metadata - .options - .to_program_settings(&root, "psycopg", &system); - - assert_eq!( - settings.search_paths.src_roots, - vec![root.clone(), root.join("psycopg")] - ); - - Ok(()) - } - - #[test] - fn no_src_root_flat_layout() -> anyhow::Result<()> { - let system = TestSystem::default(); - let root = SystemPathBuf::from("/app"); - - system - .memory_file_system() - .write_file_all( - root.join("my_package/main.py"), - r#" - print("Hello, world!") - "#, - ) - .context("Failed to write file")?; - - let metadata = ProjectMetadata::discover(&root, &system)?; - let settings = metadata - .options - .to_program_settings(&root, "my_package", &system); - - assert_eq!(settings.search_paths.src_roots, vec![root]); - - Ok(()) - } - - #[test] - fn src_root_with_tests() -> anyhow::Result<()> { - let system = TestSystem::default(); - let root = SystemPathBuf::from("/app"); - - // pytest will find `tests/test_foo.py` and realize it is NOT part of a package - // given that there's no `__init__.py` file in the same folder. - // It will then add `tests` to `sys.path` - // in order to import `test_foo.py` as the module `test_foo`. - system - .memory_file_system() - .write_files_all([ - (root.join("src/main.py"), ""), - (root.join("tests/conftest.py"), ""), - (root.join("tests/test_foo.py"), ""), - ]) - .context("Failed to write files")?; - - let metadata = ProjectMetadata::discover(&root, &system)?; - let settings = metadata - .options - .to_program_settings(&root, "my_package", &system); - - assert_eq!( - settings.search_paths.src_roots, - vec![root.clone(), root.join("src"), root.join("tests")] - ); - - // If `tests/__init__.py` is present, it is considered a package and `tests` is not added to `sys.path`. - system - .memory_file_system() - .write_file(root.join("tests/__init__.py"), "") - .context("Failed to write tests/__init__.py")?; - let metadata = ProjectMetadata::discover(&root, &system)?; - let settings = metadata - .options - .to_program_settings(&root, "my_package", &system); - - assert_eq!( - settings.search_paths.src_roots, - vec![root.clone(), root.join("src")] - ); - - Ok(()) - } - #[track_caller] fn assert_error_eq(error: &ProjectMetadataError, message: &str) { assert_eq!(error.to_string().replace('\\', "/"), message); diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index 1845790fa2..6d7ba69819 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -14,6 +14,7 @@ use ruff_db::diagnostic::{ }; use ruff_db::files::system_path_to_file; use ruff_db::system::{System, SystemPath, SystemPathBuf}; +use ruff_db::vendored::VendoredFileSystem; use ruff_macros::{Combine, OptionsMetadata, RustDoc}; use ruff_options_metadata::{OptionSet, OptionsMetadata, Visit}; use ruff_python_ast::PythonVersion; @@ -28,7 +29,8 @@ use thiserror::Error; use ty_python_semantic::lint::{GetLintError, Level, LintSource, RuleSelection}; use ty_python_semantic::{ ProgramSettings, PythonPath, PythonPlatform, PythonVersionFileSource, PythonVersionSource, - PythonVersionWithSource, SearchPathSettings, SysPrefixPathOrigin, + PythonVersionWithSource, SearchPathSettings, SearchPathValidationError, SearchPaths, + SysPrefixPathOrigin, }; use super::settings::{Override, Settings, TerminalSettings}; @@ -104,10 +106,11 @@ impl Options { project_root: &SystemPath, project_name: &str, system: &dyn System, - ) -> ProgramSettings { + vendored: &VendoredFileSystem, + ) -> anyhow::Result { let environment = self.environment.or_default(); - let python_version = + let options_python_version = environment .python_version .as_ref() @@ -120,6 +123,7 @@ impl Options { ), }, }); + let python_platform = environment .python_platform .as_deref() @@ -129,19 +133,36 @@ impl Options { tracing::info!("Defaulting to python-platform `{default}`"); default }); - ProgramSettings { + + let search_paths = self.to_search_paths(project_root, project_name, system, vendored)?; + + let python_version = options_python_version + .or_else(|| { + search_paths + .try_resolve_installation_python_version() + .map(Cow::into_owned) + }) + .unwrap_or_default(); + + tracing::info!( + "Python version: Python {python_version}, platform: {python_platform}", + python_version = python_version.version + ); + + Ok(ProgramSettings { python_version, python_platform, - search_paths: self.to_search_path_settings(project_root, project_name, system), - } + search_paths, + }) } - fn to_search_path_settings( + fn to_search_paths( &self, project_root: &SystemPath, project_name: &str, system: &dyn System, - ) -> SearchPathSettings { + vendored: &VendoredFileSystem, + ) -> Result { let environment = self.environment.or_default(); let src = self.src.or_default(); @@ -197,7 +218,7 @@ impl Options { roots }; - SearchPathSettings { + let settings = SearchPathSettings { extra_paths: environment .extra_paths .as_deref() @@ -239,7 +260,9 @@ impl Options { SysPrefixPathOrigin::LocalVenv, ) }), - } + }; + + settings.to_search_paths(system, vendored) } pub(crate) fn to_settings( diff --git a/crates/ty_project/src/metadata/value.rs b/crates/ty_project/src/metadata/value.rs index 1ad5b58e87..09d457d875 100644 --- a/crates/ty_project/src/metadata/value.rs +++ b/crates/ty_project/src/metadata/value.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Deserializer}; use std::cell::RefCell; use std::cmp::Ordering; use std::fmt; +use std::fmt::Formatter; use std::hash::{Hash, Hasher}; use std::ops::{Deref, DerefMut}; use std::sync::Arc; @@ -360,6 +361,12 @@ impl RelativePathBuf { } } +impl fmt::Display for RelativePathBuf { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + #[derive( Debug, Clone, diff --git a/crates/ty_python_semantic/src/db.rs b/crates/ty_python_semantic/src/db.rs index 9e37ed3ca9..50d0c9faad 100644 --- a/crates/ty_python_semantic/src/db.rs +++ b/crates/ty_python_semantic/src/db.rs @@ -183,15 +183,16 @@ pub(crate) mod tests { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: self.python_version, source: PythonVersionSource::default(), - }), + }, python_platform: self.python_platform, - search_paths: SearchPathSettings::new(vec![src_root]), + search_paths: SearchPathSettings::new(vec![src_root]) + .to_search_paths(db.system(), db.vendored()) + .context("Invalid search path settings")?, }, - ) - .context("Failed to configure Program settings")?; + ); Ok(db) } diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index ad2562a9df..30db10dd9f 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -6,7 +6,10 @@ use crate::lint::{LintRegistry, LintRegistryBuilder}; use crate::suppression::{INVALID_IGNORE_COMMENT, UNKNOWN_RULE, UNUSED_IGNORE_COMMENT}; pub use db::Db; pub use module_name::ModuleName; -pub use module_resolver::{KnownModule, Module, resolve_module, system_module_search_paths}; +pub use module_resolver::{ + KnownModule, Module, SearchPathValidationError, SearchPaths, resolve_module, + system_module_search_paths, +}; pub use program::{ Program, ProgramSettings, PythonPath, PythonVersionFileSource, PythonVersionSource, PythonVersionWithSource, SearchPathSettings, diff --git a/crates/ty_python_semantic/src/module_resolver/mod.rs b/crates/ty_python_semantic/src/module_resolver/mod.rs index e0e989945b..b041ea14ab 100644 --- a/crates/ty_python_semantic/src/module_resolver/mod.rs +++ b/crates/ty_python_semantic/src/module_resolver/mod.rs @@ -1,8 +1,10 @@ use std::iter::FusedIterator; pub use module::{KnownModule, Module}; +pub use path::SearchPathValidationError; +pub use resolver::SearchPaths; +pub(crate) use resolver::file_to_module; pub use resolver::resolve_module; -pub(crate) use resolver::{SearchPaths, file_to_module}; use ruff_db::system::SystemPath; use crate::Db; diff --git a/crates/ty_python_semantic/src/module_resolver/path.rs b/crates/ty_python_semantic/src/module_resolver/path.rs index 7410e7a748..c128a233f9 100644 --- a/crates/ty_python_semantic/src/module_resolver/path.rs +++ b/crates/ty_python_semantic/src/module_resolver/path.rs @@ -9,7 +9,6 @@ use ruff_db::system::{System, SystemPath, SystemPathBuf}; use ruff_db::vendored::{VendoredPath, VendoredPathBuf}; use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult, typeshed_versions}; -use crate::db::Db; use crate::module_name::ModuleName; use crate::module_resolver::resolver::ResolverContext; use crate::site_packages::SitePackagesDiscoveryError; @@ -325,62 +324,37 @@ fn query_stdlib_version( /// If validation fails for a search path derived from the user settings, /// a message must be displayed to the user, /// as type checking cannot be done reliably in these circumstances. -#[derive(Debug)] -pub(crate) enum SearchPathValidationError { +#[derive(Debug, thiserror::Error)] +pub enum SearchPathValidationError { /// The path provided by the user was not a directory + #[error("{0} does not point to a directory")] NotADirectory(SystemPathBuf), /// The path provided by the user is a directory, /// but no `stdlib/` subdirectory exists. /// (This is only relevant for stdlib search paths.) + #[error("The directory at {0} has no `stdlib/` subdirectory")] NoStdlibSubdirectory(SystemPathBuf), /// The typeshed path provided by the user is a directory, /// but `stdlib/VERSIONS` could not be read. /// (This is only relevant for stdlib search paths.) + #[error("Failed to read the custom typeshed versions file '{path}'")] FailedToReadVersionsFile { path: SystemPathBuf, + #[source] error: std::io::Error, }, /// The path provided by the user is a directory, /// and a `stdlib/VERSIONS` file exists, but it fails to parse. /// (This is only relevant for stdlib search paths.) + #[error(transparent)] VersionsParseError(TypeshedVersionsParseError), /// Failed to discover the site-packages for the configured virtual environment. - SitePackagesDiscovery(SitePackagesDiscoveryError), -} - -impl fmt::Display for SearchPathValidationError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::NotADirectory(path) => write!(f, "{path} does not point to a directory"), - Self::NoStdlibSubdirectory(path) => { - write!(f, "The directory at {path} has no `stdlib/` subdirectory") - } - Self::FailedToReadVersionsFile { path, error } => { - write!( - f, - "Failed to read the custom typeshed versions file '{path}': {error}" - ) - } - Self::VersionsParseError(underlying_error) => underlying_error.fmt(f), - SearchPathValidationError::SitePackagesDiscovery(error) => { - write!(f, "Failed to discover the site-packages directory: {error}") - } - } - } -} - -impl std::error::Error for SearchPathValidationError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - if let Self::VersionsParseError(underlying_error) = self { - Some(underlying_error) - } else { - None - } - } + #[error("Failed to discover the site-packages directory")] + SitePackagesDiscovery(#[source] SitePackagesDiscoveryError), } impl From for SearchPathValidationError { @@ -459,8 +433,10 @@ impl SearchPath { } /// Create a new standard-library search path pointing to a custom directory on disk - pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: &SystemPath) -> SearchPathResult { - let system = db.system(); + pub(crate) fn custom_stdlib( + system: &dyn System, + typeshed: &SystemPath, + ) -> SearchPathResult { if !system.is_directory(typeshed) { return Err(SearchPathValidationError::NotADirectory( typeshed.to_path_buf(), @@ -528,6 +504,10 @@ impl SearchPath { ) } + pub(crate) fn is_first_party(&self) -> bool { + matches!(&*self.0, SearchPathInner::FirstParty(_)) + } + fn is_valid_extension(&self, extension: &str) -> bool { if self.is_standard_library() { extension == "pyi" @@ -707,7 +687,7 @@ mod tests { .build(); assert_eq!( - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) + SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()) .unwrap() .to_module_path() .with_py_extension(), @@ -715,7 +695,7 @@ mod tests { ); assert_eq!( - &SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) + &SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()) .unwrap() .join("foo") .with_pyi_extension(), @@ -826,7 +806,7 @@ mod tests { let TestCase { db, stdlib, .. } = TestCaseBuilder::new() .with_mocked_typeshed(MockedTypeshed::default()) .build(); - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) + SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()) .unwrap() .to_module_path() .push("bar.py"); @@ -838,7 +818,7 @@ mod tests { let TestCase { db, stdlib, .. } = TestCaseBuilder::new() .with_mocked_typeshed(MockedTypeshed::default()) .build(); - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) + SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()) .unwrap() .to_module_path() .push("bar.rs"); @@ -870,7 +850,7 @@ mod tests { .with_mocked_typeshed(MockedTypeshed::default()) .build(); - let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap(); + let root = SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()).unwrap(); // Must have a `.pyi` extension or no extension: let bad_absolute_path = SystemPath::new("foo/stdlib/x.py"); @@ -918,7 +898,7 @@ mod tests { .with_mocked_typeshed(typeshed) .with_python_version(python_version) .build(); - let stdlib = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap(); + let stdlib = SearchPath::custom_stdlib(db.system(), stdlib.parent().unwrap()).unwrap(); (db, stdlib) } diff --git a/crates/ty_python_semantic/src/module_resolver/resolver.rs b/crates/ty_python_semantic/src/module_resolver/resolver.rs index 0a5a7eb808..2511fc3fe4 100644 --- a/crates/ty_python_semantic/src/module_resolver/resolver.rs +++ b/crates/ty_python_semantic/src/module_resolver/resolver.rs @@ -142,9 +142,9 @@ pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator { Program::get(db).search_paths(db).iter(db) } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SearchPaths { - /// Search paths that have been statically determined purely from reading Ruff's configuration settings. + /// Search paths that have been statically determined purely from reading ty's configuration settings. /// These shouldn't ever change unless the config settings themselves change. static_paths: Vec, @@ -174,8 +174,9 @@ impl SearchPaths { /// /// [module resolution order]: https://typing.python.org/en/latest/spec/distributing.html#import-resolution-ordering pub(crate) fn from_settings( - db: &dyn Db, settings: &SearchPathSettings, + system: &dyn System, + vendored: &VendoredFileSystem, ) -> Result { fn canonicalize(path: &SystemPath, system: &dyn System) -> SystemPathBuf { system @@ -190,14 +191,10 @@ impl SearchPaths { python_path, } = settings; - let system = db.system(); - let files = db.files(); - let mut static_paths = vec![]; for path in extra_paths { let path = canonicalize(path, system); - files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath); tracing::debug!("Adding extra search-path '{path}'"); static_paths.push(SearchPath::extra(system, path)?); @@ -212,8 +209,6 @@ impl SearchPaths { let typeshed = canonicalize(typeshed, system); tracing::debug!("Adding custom-stdlib search path '{typeshed}'"); - files.try_add_root(db.upcast(), &typeshed, FileRootKind::LibrarySearchPath); - let versions_path = typeshed.join("stdlib/VERSIONS"); let versions_content = system.read_to_string(&versions_path).map_err(|error| { @@ -225,13 +220,13 @@ impl SearchPaths { let parsed: TypeshedVersions = versions_content.parse()?; - let search_path = SearchPath::custom_stdlib(db, &typeshed)?; + let search_path = SearchPath::custom_stdlib(system, &typeshed)?; (parsed, search_path) } else { tracing::debug!("Using vendored stdlib"); ( - vendored_typeshed_versions(db), + vendored_typeshed_versions(vendored), SearchPath::vendored_stdlib(), ) }; @@ -282,7 +277,6 @@ impl SearchPaths { for path in site_packages_paths { tracing::debug!("Adding site-packages search path '{path}'"); - files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath); site_packages.push(SearchPath::site_packages(system, path)?); } @@ -313,6 +307,17 @@ impl SearchPaths { }) } + pub(crate) fn try_register_static_roots(&self, db: &dyn Db) { + let files = db.files(); + for path in self.static_paths.iter().chain(self.site_packages.iter()) { + if let Some(system_path) = path.as_system_path() { + if !path.is_first_party() { + files.try_add_root(db.upcast(), system_path, FileRootKind::LibrarySearchPath); + } + } + } + } + pub(super) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> { SearchPathIterator { db, @@ -1482,20 +1487,20 @@ mod tests { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: PythonVersion::PY38, source: PythonVersionSource::default(), - }), + }, python_platform: PythonPlatform::default(), search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![src.clone()], custom_typeshed: Some(custom_typeshed), python_path: PythonPath::KnownSitePackages(vec![site_packages]), - }, + ..SearchPathSettings::new(vec![src.clone()]) + } + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"), }, - ) - .context("Invalid program settings")?; + ); let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); let bar_module = resolve_module(&db, &ModuleName::new_static("bar").unwrap()).unwrap(); @@ -2001,20 +2006,19 @@ not_a_directory Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![SystemPathBuf::from("/src")], - custom_typeshed: None, python_path: PythonPath::KnownSitePackages(vec![ venv_site_packages, system_site_packages, ]), - }, + ..SearchPathSettings::new(vec![SystemPathBuf::from("/src")]) + } + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"), }, - ) - .expect("Valid program settings"); + ); // The editable installs discovered from the `.pth` file in the first `site-packages` directory // take precedence over the second `site-packages` directory... @@ -2080,17 +2084,13 @@ not_a_directory Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![src], - custom_typeshed: None, - python_path: PythonPath::KnownSitePackages(vec![]), - }, + search_paths: SearchPathSettings::new(vec![src]) + .to_search_paths(db.system(), db.vendored()) + .expect("valid search path settings"), }, - ) - .expect("Valid program settings"); + ); // Now try to resolve the module `A` (note the capital `A` instead of `a`). let a_module_name = ModuleName::new_static("A").unwrap(); @@ -2123,17 +2123,16 @@ not_a_directory Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![project_directory], - custom_typeshed: None, python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]), - }, + ..SearchPathSettings::new(vec![project_directory]) + } + .to_search_paths(db.system(), db.vendored()) + .unwrap(), }, - ) - .unwrap(); + ); let foo_module_file = File::new(&db, FilePath::System(installed_foo_module)); let module = file_to_module(&db, foo_module_file).unwrap(); diff --git a/crates/ty_python_semantic/src/module_resolver/testing.rs b/crates/ty_python_semantic/src/module_resolver/testing.rs index 57f366c28c..f2d70ee894 100644 --- a/crates/ty_python_semantic/src/module_resolver/testing.rs +++ b/crates/ty_python_semantic/src/module_resolver/testing.rs @@ -1,3 +1,4 @@ +use ruff_db::Db; use ruff_db::system::{ DbWithTestSystem as _, DbWithWritableSystem as _, SystemPath, SystemPathBuf, }; @@ -237,20 +238,20 @@ impl TestCaseBuilder { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: python_version, source: PythonVersionSource::default(), - }), + }, python_platform, search_paths: SearchPathSettings { - extra_paths: vec![], - src_roots: vec![src.clone()], custom_typeshed: Some(typeshed.clone()), python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]), - }, + ..SearchPathSettings::new(vec![src.clone()]) + } + .to_search_paths(db.system(), db.vendored()) + .expect("valid search path settings"), }, - ) - .expect("Valid program settings"); + ); TestCase { db, @@ -298,18 +299,19 @@ impl TestCaseBuilder { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: python_version, source: PythonVersionSource::default(), - }), + }, python_platform, search_paths: SearchPathSettings { python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]), ..SearchPathSettings::new(vec![src.clone()]) - }, + } + .to_search_paths(db.system(), db.vendored()) + .expect("valid search path settings"), }, - ) - .expect("Valid search path settings"); + ); TestCase { db, diff --git a/crates/ty_python_semantic/src/module_resolver/typeshed.rs b/crates/ty_python_semantic/src/module_resolver/typeshed.rs index 2d5ace3f2d..432c9bd4fa 100644 --- a/crates/ty_python_semantic/src/module_resolver/typeshed.rs +++ b/crates/ty_python_semantic/src/module_resolver/typeshed.rs @@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; +use ruff_db::vendored::VendoredFileSystem; use ruff_python_ast::{PythonVersion, PythonVersionDeserializationError}; use rustc_hash::FxHashMap; @@ -11,9 +12,11 @@ use crate::Program; use crate::db::Db; use crate::module_name::ModuleName; -pub(in crate::module_resolver) fn vendored_typeshed_versions(db: &dyn Db) -> TypeshedVersions { +pub(in crate::module_resolver) fn vendored_typeshed_versions( + vendored: &VendoredFileSystem, +) -> TypeshedVersions { TypeshedVersions::from_str( - &db.vendored() + &vendored .read_to_string("stdlib/VERSIONS") .expect("The vendored typeshed stubs should contain a VERSIONS file"), ) @@ -25,7 +28,7 @@ pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions { } #[derive(Debug, PartialEq, Eq, Clone)] -pub(crate) struct TypeshedVersionsParseError { +pub struct TypeshedVersionsParseError { line_number: Option, reason: TypeshedVersionsParseErrorKind, } @@ -71,7 +74,7 @@ pub(crate) enum TypeshedVersionsParseErrorKind { VersionParseError(#[from] PythonVersionDeserializationError), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct TypeshedVersions(FxHashMap); impl TypeshedVersions { @@ -305,11 +308,8 @@ mod tests { use std::num::{IntErrorKind, NonZeroU16}; use std::path::Path; - use insta::assert_snapshot; - - use crate::db::tests::TestDb; - use super::*; + use insta::assert_snapshot; const TYPESHED_STDLIB_DIR: &str = "stdlib"; @@ -329,9 +329,7 @@ mod tests { #[test] fn can_parse_vendored_versions_file() { - let db = TestDb::new(); - - let versions = vendored_typeshed_versions(&db); + let versions = vendored_typeshed_versions(ty_vendored::file_system()); assert!(versions.len() > 100); assert!(versions.len() < 1000); @@ -368,8 +366,7 @@ mod tests { #[test] fn typeshed_versions_consistent_with_vendored_stubs() { - let db = TestDb::new(); - let vendored_typeshed_versions = vendored_typeshed_versions(&db); + let vendored_typeshed_versions = vendored_typeshed_versions(ty_vendored::file_system()); let vendored_typeshed_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../ty_vendored/vendor/typeshed"); diff --git a/crates/ty_python_semantic/src/program.rs b/crates/ty_python_semantic/src/program.rs index 6facef1de7..4bf2297357 100644 --- a/crates/ty_python_semantic/src/program.rs +++ b/crates/ty_python_semantic/src/program.rs @@ -1,15 +1,14 @@ -use std::borrow::Cow; use std::sync::Arc; use crate::Db; -use crate::module_resolver::SearchPaths; +use crate::module_resolver::{SearchPathValidationError, SearchPaths}; use crate::python_platform::PythonPlatform; use crate::site_packages::SysPrefixPathOrigin; -use anyhow::Context; use ruff_db::diagnostic::Span; use ruff_db::files::system_path_to_file; -use ruff_db::system::{SystemPath, SystemPathBuf}; +use ruff_db::system::{System, SystemPath, SystemPathBuf}; +use ruff_db::vendored::VendoredFileSystem; use ruff_python_ast::PythonVersion; use ruff_text_size::TextRange; use salsa::Durability; @@ -28,66 +27,44 @@ pub struct Program { } impl Program { - pub fn from_settings(db: &dyn Db, settings: ProgramSettings) -> anyhow::Result { + pub fn init_or_update(db: &mut dyn Db, settings: ProgramSettings) -> Self { + match Self::try_get(db) { + Some(program) => { + program.update_from_settings(db, settings); + program + } + None => Self::from_settings(db, settings), + } + } + + pub fn from_settings(db: &dyn Db, settings: ProgramSettings) -> Self { let ProgramSettings { - python_version: python_version_with_source, + python_version, python_platform, search_paths, } = settings; - let search_paths = SearchPaths::from_settings(db, &search_paths) - .with_context(|| "Invalid search path settings")?; + search_paths.try_register_static_roots(db); - let python_version_with_source = - Self::resolve_python_version(python_version_with_source, &search_paths); - - tracing::info!( - "Python version: Python {python_version}, platform: {python_platform}", - python_version = python_version_with_source.version - ); - - Ok( - Program::builder(python_version_with_source, python_platform, search_paths) - .durability(Durability::HIGH) - .new(db), - ) + Program::builder(python_version, python_platform, search_paths) + .durability(Durability::HIGH) + .new(db) } pub fn python_version(self, db: &dyn Db) -> PythonVersion { self.python_version_with_source(db).version } - fn resolve_python_version( - config_value: Option, - search_paths: &SearchPaths, - ) -> PythonVersionWithSource { - config_value - .or_else(|| { - search_paths - .try_resolve_installation_python_version() - .map(Cow::into_owned) - }) - .unwrap_or_default() - } - - pub fn update_from_settings( - self, - db: &mut dyn Db, - settings: ProgramSettings, - ) -> anyhow::Result<()> { + pub fn update_from_settings(self, db: &mut dyn Db, settings: ProgramSettings) { let ProgramSettings { - python_version: python_version_with_source, + python_version, python_platform, search_paths, } = settings; - let search_paths = SearchPaths::from_settings(db, &search_paths)?; - - let new_python_version = - Self::resolve_python_version(python_version_with_source, &search_paths); - if self.search_paths(db) != &search_paths { tracing::debug!("Updating search paths"); + search_paths.try_register_static_roots(db); self.set_search_paths(db).to(search_paths); } @@ -96,48 +73,13 @@ impl Program { self.set_python_platform(db).to(python_platform); } - if &new_python_version != self.python_version_with_source(db) { + if &python_version != self.python_version_with_source(db) { tracing::debug!( "Updating python version: Python {version}", - version = new_python_version.version + version = python_version.version ); - self.set_python_version_with_source(db) - .to(new_python_version); + self.set_python_version_with_source(db).to(python_version); } - - Ok(()) - } - - /// Update the search paths for the program. - pub fn update_search_paths( - self, - db: &mut dyn Db, - search_path_settings: &SearchPathSettings, - ) -> anyhow::Result<()> { - let search_paths = SearchPaths::from_settings(db, search_path_settings)?; - - let current_python_version = self.python_version_with_source(db); - - let python_version_from_environment = search_paths - .try_resolve_installation_python_version() - .map(Cow::into_owned) - .unwrap_or_default(); - - if current_python_version != &python_version_from_environment - && current_python_version.source.priority() - <= python_version_from_environment.source.priority() - { - tracing::debug!("Updating Python version from environment"); - self.set_python_version_with_source(db) - .to(python_version_from_environment); - } - - if self.search_paths(db) != &search_paths { - tracing::debug!("Updating search paths"); - self.set_search_paths(db).to(search_paths); - } - - Ok(()) } pub fn custom_stdlib_search_path(self, db: &dyn Db) -> Option<&SystemPath> { @@ -147,9 +89,9 @@ impl Program { #[derive(Clone, Debug, Eq, PartialEq)] pub struct ProgramSettings { - pub python_version: Option, + pub python_version: PythonVersionWithSource, pub python_platform: PythonPlatform, - pub search_paths: SearchPathSettings, + pub search_paths: SearchPaths, } #[derive(Clone, Debug, Eq, PartialEq, Default)] @@ -177,35 +119,6 @@ pub enum PythonVersionSource { Default, } -impl PythonVersionSource { - fn priority(&self) -> PythonSourcePriority { - match self { - PythonVersionSource::Default => PythonSourcePriority::Default, - PythonVersionSource::PyvenvCfgFile(_) => PythonSourcePriority::PyvenvCfgFile, - PythonVersionSource::ConfigFile(_) => PythonSourcePriority::ConfigFile, - PythonVersionSource::Cli => PythonSourcePriority::Cli, - PythonVersionSource::InstallationDirectoryLayout { .. } => { - PythonSourcePriority::InstallationDirectoryLayout - } - } - } -} - -/// The priority in which Python version sources are considered. -/// The lower down the variant appears in this enum, the higher its priority. -/// -/// For example, if a Python version is specified in a pyproject.toml file -/// but *also* via a CLI argument, the CLI argument will take precedence. -#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] -#[cfg_attr(test, derive(strum_macros::EnumIter))] -enum PythonSourcePriority { - Default, - InstallationDirectoryLayout, - PyvenvCfgFile, - ConfigFile, - Cli, -} - /// Information regarding the file and [`TextRange`] of the configuration /// from which we inferred the Python version. #[derive(Debug, PartialEq, Eq, Clone)] @@ -270,11 +183,26 @@ impl SearchPathSettings { pub fn new(src_roots: Vec) -> Self { Self { src_roots, + ..SearchPathSettings::empty() + } + } + + pub fn empty() -> Self { + SearchPathSettings { + src_roots: vec![], extra_paths: vec![], custom_typeshed: None, python_path: PythonPath::KnownSitePackages(vec![]), } } + + pub fn to_search_paths( + &self, + system: &dyn System, + vendored: &VendoredFileSystem, + ) -> Result { + SearchPaths::from_settings(self, system, vendored) + } } #[derive(Debug, Clone, Eq, PartialEq)] @@ -308,74 +236,3 @@ impl PythonPath { Self::IntoSysPrefix(path.into(), origin) } } - -#[cfg(test)] -mod tests { - use super::*; - use strum::IntoEnumIterator; - - #[test] - fn test_python_version_source_priority() { - for priority in PythonSourcePriority::iter() { - match priority { - // CLI source takes priority over all other sources. - PythonSourcePriority::Cli => { - for other in PythonSourcePriority::iter() { - assert!(priority >= other, "{other:?}"); - } - } - // Config files have lower priority than CLI arguments, - // but higher than pyvenv.cfg files and the fallback default. - PythonSourcePriority::ConfigFile => { - for other in PythonSourcePriority::iter() { - match other { - PythonSourcePriority::Cli => assert!(other > priority, "{other:?}"), - PythonSourcePriority::ConfigFile => assert_eq!(priority, other), - PythonSourcePriority::PyvenvCfgFile - | PythonSourcePriority::Default - | PythonSourcePriority::InstallationDirectoryLayout => { - assert!(priority > other, "{other:?}"); - } - } - } - } - // Pyvenv.cfg files have lower priority than CLI flags and config files, - // but higher than the default fallback. - PythonSourcePriority::PyvenvCfgFile => { - for other in PythonSourcePriority::iter() { - match other { - PythonSourcePriority::Cli | PythonSourcePriority::ConfigFile => { - assert!(other > priority, "{other:?}"); - } - PythonSourcePriority::PyvenvCfgFile => assert_eq!(priority, other), - PythonSourcePriority::Default - | PythonSourcePriority::InstallationDirectoryLayout => { - assert!(priority > other, "{other:?}"); - } - } - } - } - PythonSourcePriority::InstallationDirectoryLayout => { - for other in PythonSourcePriority::iter() { - match other { - PythonSourcePriority::Cli - | PythonSourcePriority::ConfigFile - | PythonSourcePriority::PyvenvCfgFile => { - assert!(other > priority, "{other:?}"); - } - PythonSourcePriority::InstallationDirectoryLayout => { - assert_eq!(priority, other); - } - PythonSourcePriority::Default => assert!(priority > other, "{other:?}"), - } - } - } - PythonSourcePriority::Default => { - for other in PythonSourcePriority::iter() { - assert!(priority <= other, "{other:?}"); - } - } - } - } - } -} diff --git a/crates/ty_python_semantic/src/site_packages.rs b/crates/ty_python_semantic/src/site_packages.rs index 8a3e91b864..9c353e8ee6 100644 --- a/crates/ty_python_semantic/src/site_packages.rs +++ b/crates/ty_python_semantic/src/site_packages.rs @@ -532,7 +532,7 @@ impl SystemEnvironment { /// Enumeration of ways in which `site-packages` discovery can fail. #[derive(Debug)] -pub(crate) enum SitePackagesDiscoveryError { +pub enum SitePackagesDiscoveryError { /// `site-packages` discovery failed because the provided path couldn't be canonicalized. CanonicalizationError(SystemPathBuf, SysPrefixPathOrigin, io::Error), @@ -698,7 +698,7 @@ fn display_error( /// The various ways in which parsing a `pyvenv.cfg` file could fail #[derive(Debug)] -pub(crate) enum PyvenvCfgParseErrorKind { +pub enum PyvenvCfgParseErrorKind { MalformedKeyValuePair { line_number: NonZeroUsize }, NoHomeKey, InvalidHomeValue(io::Error), @@ -853,7 +853,7 @@ fn site_packages_directory_from_sys_prefix( /// /// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix #[derive(Debug, PartialEq, Eq, Clone)] -pub(crate) struct SysPrefixPath { +pub struct SysPrefixPath { inner: SystemPathBuf, origin: SysPrefixPathOrigin, } diff --git a/crates/ty_python_semantic/tests/corpus.rs b/crates/ty_python_semantic/tests/corpus.rs index 2f72a4bce3..6136fd4d74 100644 --- a/crates/ty_python_semantic/tests/corpus.rs +++ b/crates/ty_python_semantic/tests/corpus.rs @@ -1,8 +1,8 @@ use anyhow::{Context, anyhow}; -use ruff_db::Upcast; use ruff_db::files::{File, Files, system_path_to_file}; use ruff_db::system::{DbWithTestSystem, System, SystemPath, SystemPathBuf, TestSystem}; use ruff_db::vendored::VendoredFileSystem; +use ruff_db::{Db, Upcast}; use ruff_python_ast::PythonVersion; use ty_python_semantic::lint::{LintRegistry, RuleSelection}; @@ -205,15 +205,16 @@ impl CorpusDb { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: PythonVersion::latest_ty(), source: PythonVersionSource::default(), - }), + }, python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(vec![]), + search_paths: SearchPathSettings::new(vec![]) + .to_search_paths(db.system(), db.vendored()) + .unwrap(), }, - ) - .unwrap(); + ); db } diff --git a/crates/ty_test/src/assertion.rs b/crates/ty_test/src/assertion.rs index fd567dc60a..348d96ec09 100644 --- a/crates/ty_test/src/assertion.rs +++ b/crates/ty_test/src/assertion.rs @@ -489,8 +489,8 @@ pub(crate) enum ErrorAssertionParseError<'a> { #[cfg(test)] mod tests { use super::*; - use ruff_db::files::system_path_to_file; use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::{Db as _, files::system_path_to_file}; use ruff_python_trivia::textwrap::dedent; use ruff_source_file::OneIndexed; use ty_python_semantic::{ @@ -501,15 +501,13 @@ mod tests { let mut db = Db::setup(); let settings = ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(Vec::new()), + search_paths: SearchPathSettings::new(Vec::new()) + .to_search_paths(db.system(), db.vendored()) + .unwrap(), }; - match Program::try_get(&db) { - Some(program) => program.update_from_settings(&mut db, settings), - None => Program::from_settings(&db, settings).map(|_| ()), - } - .expect("Failed to update Program settings in TestDb"); + Program::init_or_update(&mut db, settings); db.write_file("/src/test.py", source).unwrap(); let file = system_path_to_file(&db, "/src/test.py").unwrap(); diff --git a/crates/ty_test/src/lib.rs b/crates/ty_test/src/lib.rs index f9e779fd0e..01c2ff4e94 100644 --- a/crates/ty_test/src/lib.rs +++ b/crates/ty_test/src/lib.rs @@ -5,7 +5,6 @@ use camino::Utf8Path; use colored::Colorize; use config::SystemKind; use parser as test_parser; -use ruff_db::Upcast; use ruff_db::diagnostic::{ Diagnostic, DisplayDiagnosticConfig, create_parse_diagnostic, create_unsupported_syntax_diagnostic, @@ -15,6 +14,7 @@ use ruff_db::panic::catch_unwind; use ruff_db::parsed::parsed_module; use ruff_db::system::{DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_db::testing::{setup_logging, setup_logging_with_filter}; +use ruff_db::{Db as _, Upcast}; use ruff_source_file::{LineIndex, OneIndexed}; use std::backtrace::BacktraceStatus; use std::fmt::Write; @@ -260,10 +260,10 @@ fn run_test( let configuration = test.configuration(); let settings = ProgramSettings { - python_version: Some(PythonVersionWithSource { + python_version: PythonVersionWithSource { version: python_version, source: PythonVersionSource::Cli, - }), + }, python_platform: configuration .python_platform() .unwrap_or(PythonPlatform::Identifier("linux".to_string())), @@ -280,14 +280,12 @@ fn run_test( ) }) .unwrap_or(PythonPath::KnownSitePackages(vec![])), - }, + } + .to_search_paths(db.system(), db.vendored()) + .expect("Failed to resolve search path settings"), }; - match Program::try_get(db) { - Some(program) => program.update_from_settings(db, settings), - None => Program::from_settings(db, settings).map(|_| ()), - } - .expect("Failed to update Program settings in TestDb"); + Program::init_or_update(db, settings); // When snapshot testing is enabled, this is populated with // all diagnostics. Otherwise it remains empty. diff --git a/crates/ty_test/src/matcher.rs b/crates/ty_test/src/matcher.rs index 92e150d860..3d574242bd 100644 --- a/crates/ty_test/src/matcher.rs +++ b/crates/ty_test/src/matcher.rs @@ -338,6 +338,7 @@ impl Matcher { #[cfg(test)] mod tests { use super::FailuresByLine; + use ruff_db::Db; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, Severity, Span}; use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::DbWithWritableSystem as _; @@ -385,15 +386,13 @@ mod tests { let mut db = crate::db::Db::setup(); let settings = ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(Vec::new()), + search_paths: SearchPathSettings::new(Vec::new()) + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search paths settings"), }; - match Program::try_get(&db) { - Some(program) => program.update_from_settings(&mut db, settings), - None => Program::from_settings(&db, settings).map(|_| ()), - } - .expect("Failed to update Program settings in TestDb"); + Program::init_or_update(&mut db, settings); db.write_file("/src/test.py", source).unwrap(); let file = system_path_to_file(&db, "/src/test.py").unwrap(); diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index e20ab2a803..7025fa4a10 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -1,7 +1,6 @@ use std::any::Any; use js_sys::{Error, JsString}; -use ruff_db::Upcast; use ruff_db::diagnostic::{self, DisplayDiagnosticConfig}; use ruff_db::files::{File, FileRange, system_path_to_file}; use ruff_db::source::{line_index, source_text}; @@ -10,6 +9,7 @@ use ruff_db::system::{ CaseSensitivity, DirectoryEntry, GlobError, MemoryFileSystem, Metadata, PatternError, System, SystemPath, SystemPathBuf, SystemVirtualPath, }; +use ruff_db::{Db as _, Upcast}; use ruff_notebook::Notebook; use ruff_python_formatter::formatted_file; use ruff_source_file::{LineIndex, OneIndexed, SourceLocation}; @@ -97,10 +97,10 @@ impl Workspace { ) .map_err(into_error)?; - let program_settings = project.to_program_settings(&self.system); - Program::get(&self.db) - .update_from_settings(&mut self.db, program_settings) + let program_settings = project + .to_program_settings(&self.system, self.db.vendored()) .map_err(into_error)?; + Program::get(&self.db).update_from_settings(&mut self.db, program_settings); self.db.project().reload(&mut self.db, project); diff --git a/fuzz/fuzz_targets/ty_check_invalid_syntax.rs b/fuzz/fuzz_targets/ty_check_invalid_syntax.rs index 974a5ae0a9..5ab994e7eb 100644 --- a/fuzz/fuzz_targets/ty_check_invalid_syntax.rs +++ b/fuzz/fuzz_targets/ty_check_invalid_syntax.rs @@ -118,12 +118,13 @@ fn setup_db() -> TestDb { Program::from_settings( &db, ProgramSettings { - python_version: Some(PythonVersionWithSource::default()), + python_version: PythonVersionWithSource::default(), python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(vec![src_root]), + search_paths: SearchPathSettings::new(vec![src_root]) + .to_search_paths(db.system(), db.vendored()) + .expect("Valid search path settings"), }, - ) - .expect("Valid search path settings"); + ); db }