[ty] Move venv and conda env discovery to SearchPath::from_settings (#18938)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

crates/ruff/tests/analyze_graph.rs

@@ -17,6 +17,7 @@ fn command() -> Command {
     command.arg("analyze");
     command.arg("graph");
     command.arg("--preview");
+    command.env_clear();
     command
 }
 

crates/ruff_benchmark/benches/ty.rs

@@ -429,8 +429,7 @@ impl<'a> ProjectBenchmark<'a> {
         metadata.apply_options(Options {
             environment: Some(EnvironmentOptions {
                 python_version: Some(RangedValue::cli(self.project.config.python_version)),
-                python: (!self.project.config().dependencies.is_empty())
+                python: Some(RelativePathBuf::cli(SystemPath::new(".venv"))),
-                    .then_some(RelativePathBuf::cli(SystemPath::new(".venv"))),
                 ..EnvironmentOptions::default()
             }),
             ..Options::default()

crates/ruff_benchmark/benches/ty_walltime.rs

@@ -36,8 +36,7 @@ impl<'a> Benchmark<'a> {
         metadata.apply_options(Options {
             environment: Some(EnvironmentOptions {
                 python_version: Some(RangedValue::cli(self.project.config.python_version)),
-                python: (!self.project.config().dependencies.is_empty())
+                python: Some(RelativePathBuf::cli(SystemPath::new(".venv"))),
-                    .then_some(RelativePathBuf::cli(SystemPath::new(".venv"))),
                 ..EnvironmentOptions::default()
             }),
             ..Options::default()

crates/ruff_benchmark/src/real_world_projects.rs

@@ -74,19 +74,17 @@ impl<'a> RealWorldProject<'a> {
         };
 
         // Install dependencies if specified
-        if !checkout.project().dependencies.is_empty() {
+        tracing::debug!(
-            tracing::debug!(
+            "Installing {} dependencies for project '{}'...",
-                "Installing {} dependencies for project '{}'...",
+            checkout.project().dependencies.len(),
-                checkout.project().dependencies.len(),
+            checkout.project().name
-                checkout.project().name
+        );
-            );
+        let start_install = std::time::Instant::now();
-            let start = std::time::Instant::now();
+        install_dependencies(&checkout)?;
-            install_dependencies(&checkout)?;
+        tracing::debug!(
-            tracing::debug!(
+            "Dependency installation completed in {:.2}s",
-                "Dependency installation completed in {:.2}s",
+            start_install.elapsed().as_secs_f64()
-                start.elapsed().as_secs_f64()
+        );
-            );
-        }
 
         tracing::debug!("Project setup took: {:.2}s", start.elapsed().as_secs_f64());
 
@@ -281,6 +279,14 @@ fn install_dependencies(checkout: &Checkout) -> Result<()> {
         String::from_utf8_lossy(&output.stderr)
     );
 
+    if checkout.project().dependencies.is_empty() {
+        tracing::debug!(
+            "No dependencies to install for project '{}'",
+            checkout.project().name
+        );
+        return Ok(());
+    }
+
     // Install dependencies with date constraint in the isolated environment
     let mut cmd = Command::new("uv");
     cmd.args([

crates/ruff_db/Cargo.toml

@@ -30,14 +30,14 @@ filetime = { workspace = true }
 glob = { workspace = true }
 ignore = { workspace = true, optional = true }
 matchit = { workspace = true }
+path-slash = { workspace = true }
+rustc-hash = { workspace = true }
 salsa = { workspace = true }
 schemars = { workspace = true, optional = true }
 serde = { workspace = true, optional = true }
-path-slash = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true, optional = true }
-rustc-hash = { workspace = true }
 zip = { workspace = true }
 
 [target.'cfg(target_arch="wasm32")'.dependencies]

crates/ruff_graph/src/db.rs

@@ -9,7 +9,7 @@ use ruff_db::{Db as SourceDb, Upcast};
 use ruff_python_ast::PythonVersion;
 use ty_python_semantic::lint::{LintRegistry, RuleSelection};
 use ty_python_semantic::{
-    Db, Program, ProgramSettings, PythonPath, PythonPlatform, PythonVersionSource,
+    Db, Program, ProgramSettings, PythonEnvironmentPath, PythonPlatform, PythonVersionSource,
     PythonVersionWithSource, SearchPathSettings, SysPrefixPathOrigin, default_lint_registry,
 };
 
@@ -36,11 +36,11 @@ impl ModuleDb {
         venv_path: Option<SystemPathBuf>,
     ) -> Result<Self> {
         let mut search_paths = SearchPathSettings::new(src_roots);
+        // TODO: Consider setting `PythonPath::Auto` if no venv_path is provided.
         if let Some(venv_path) = venv_path {
-            search_paths.python_path =
+            search_paths.python_environment =
-                PythonPath::sys_prefix(venv_path, SysPrefixPathOrigin::PythonCliFlag);
+                PythonEnvironmentPath::explicit(venv_path, SysPrefixPathOrigin::PythonCliFlag);
         }
-
         let db = Self::default();
         let search_paths = search_paths
             .to_search_paths(db.system(), db.vendored())

crates/ty/tests/cli/main.rs

@@ -708,9 +708,8 @@ impl CliTest {
         let mut command = Command::new(get_cargo_bin("ty"));
         command.current_dir(&self.project_dir).arg("check");
 
-        // Unset environment variables that can affect test behavior
+        // Unset all environment variables because they can affect test behavior.
-        command.env_remove("VIRTUAL_ENV");
+        command.env_clear();
-        command.env_remove("CONDA_PREFIX");
 
         command
     }

crates/ty_project/src/combine.rs

@@ -3,7 +3,7 @@ use std::{collections::HashMap, hash::BuildHasher};
 use ordermap::OrderMap;
 use ruff_db::system::SystemPathBuf;
 use ruff_python_ast::PythonVersion;
-use ty_python_semantic::{PythonPath, PythonPlatform};
+use ty_python_semantic::{PythonEnvironmentPath, PythonPlatform};
 
 /// Combine two values, preferring the values in `self`.
 ///
@@ -141,7 +141,7 @@ macro_rules! impl_noop_combine {
 
 impl_noop_combine!(SystemPathBuf);
 impl_noop_combine!(PythonPlatform);
-impl_noop_combine!(PythonPath);
+impl_noop_combine!(PythonEnvironmentPath);
 impl_noop_combine!(PythonVersion);
 
 // std types

crates/ty_project/src/metadata/options.rs

@@ -2,10 +2,11 @@ use crate::Db;
 use crate::combine::Combine;
 use crate::glob::{ExcludeFilter, IncludeExcludeFilter, IncludeFilter, PortableGlobKind};
 use crate::metadata::settings::{OverrideSettings, SrcSettings};
+
+use super::settings::{Override, Settings, TerminalSettings};
 use crate::metadata::value::{
     RangedValue, RelativeGlobPattern, RelativePathBuf, ValueSource, ValueSourceGuard,
 };
-
 use ordermap::OrderMap;
 use ruff_db::RustDoc;
 use ruff_db::diagnostic::{
@@ -28,13 +29,11 @@ use std::sync::Arc;
 use thiserror::Error;
 use ty_python_semantic::lint::{GetLintError, Level, LintSource, RuleSelection};
 use ty_python_semantic::{
-    ProgramSettings, PythonPath, PythonPlatform, PythonVersionFileSource, PythonVersionSource,
+    ProgramSettings, PythonEnvironmentPath, PythonPlatform, PythonVersionFileSource,
-    PythonVersionWithSource, SearchPathSettings, SearchPathValidationError, SearchPaths,
+    PythonVersionSource, PythonVersionWithSource, SearchPathSettings, SearchPathValidationError,
-    SysPrefixPathOrigin,
+    SearchPaths, SysPrefixPathOrigin,
 };
 
-use super::settings::{Override, Settings, TerminalSettings};
-
 #[derive(
     Debug, Default, Clone, PartialEq, Eq, Combine, Serialize, Deserialize, OptionsMetadata,
 )]
@@ -231,7 +230,7 @@ impl Options {
                 .typeshed
                 .as_ref()
                 .map(|path| path.absolute(project_root, system)),
-            python_path: environment
+            python_environment: environment
                 .python
                 .as_ref()
                 .map(|python_path| {
@@ -242,24 +241,12 @@ impl Options {
                             python_path.range(),
                         ),
                     };
-                    PythonPath::sys_prefix(python_path.absolute(project_root, system), origin)
+                    PythonEnvironmentPath::explicit(
-                })
+                        python_path.absolute(project_root, system),
-                .or_else(|| {
+                        origin,
-                    system.env_var("VIRTUAL_ENV").ok().map(|virtual_env| {
-                        PythonPath::sys_prefix(virtual_env, SysPrefixPathOrigin::VirtualEnvVar)
-                    })
-                })
-                .or_else(|| {
-                    system.env_var("CONDA_PREFIX").ok().map(|path| {
-                        PythonPath::sys_prefix(path, SysPrefixPathOrigin::CondaPrefixVar)
-                    })
-                })
-                .unwrap_or_else(|| {
-                    PythonPath::sys_prefix(
-                        project_root.to_path_buf(),
-                        SysPrefixPathOrigin::LocalVenv,
                     )
-                }),
+                })
+                .unwrap_or_else(|| PythonEnvironmentPath::Discover(project_root.to_path_buf())),
         };
 
         settings.to_search_paths(system, vendored)

crates/ty_python_semantic/src/lib.rs

@@ -11,7 +11,7 @@ pub use module_resolver::{
     system_module_search_paths,
 };
 pub use program::{
-    Program, ProgramSettings, PythonPath, PythonVersionFileSource, PythonVersionSource,
+    Program, ProgramSettings, PythonEnvironmentPath, PythonVersionFileSource, PythonVersionSource,
     PythonVersionWithSource, SearchPathSettings,
 };
 pub use python_platform::PythonPlatform;

crates/ty_python_semantic/src/module_resolver/resolver.rs

@@ -15,9 +15,12 @@ use ruff_python_ast::PythonVersion;
 use crate::db::Db;
 use crate::module_name::ModuleName;
 use crate::module_resolver::typeshed::{TypeshedVersions, vendored_typeshed_versions};
-use crate::site_packages::{PythonEnvironment, SitePackagesPaths, SysPrefixPathOrigin};
+use crate::site_packages::{
+    PythonEnvironment, SitePackagesDiscoveryError, SitePackagesPaths, SysPrefixPathOrigin,
+};
 use crate::{
-    Program, PythonPath, PythonVersionSource, PythonVersionWithSource, SearchPathSettings,
+    Program, PythonEnvironmentPath, PythonVersionSource, PythonVersionWithSource,
+    SearchPathSettings,
 };
 
 use super::module::{Module, ModuleKind};
@@ -188,7 +191,7 @@ impl SearchPaths {
             extra_paths,
             src_roots,
             custom_typeshed: typeshed,
-            python_path,
+            python_environment: python_path,
         } = settings;
 
         let mut static_paths = vec![];
@@ -234,37 +237,16 @@ impl SearchPaths {
         static_paths.push(stdlib_path);
 
         let (site_packages_paths, python_version) = match python_path {
-            PythonPath::IntoSysPrefix(path, origin) => {
+            PythonEnvironmentPath::Discover(project_root) => {
-                if origin == &SysPrefixPathOrigin::LocalVenv {
+                Self::discover_python_environment(system, project_root)?
-                    tracing::debug!("Discovering virtual environment in `{path}`");
-                    let virtual_env_directory = path.join(".venv");
-
-                    PythonEnvironment::new(
-                        &virtual_env_directory,
-                        SysPrefixPathOrigin::LocalVenv,
-                        system,
-                    )
-                    .and_then(|venv| venv.into_settings(system))
-                    .inspect_err(|err| {
-                        if system.is_directory(&virtual_env_directory) {
-                            tracing::debug!(
-                                "Ignoring automatically detected virtual environment at `{}`: {}",
-                                &virtual_env_directory,
-                                err
-                            );
-                        }
-                    })
-                    .unwrap_or_else(|_| {
-                        tracing::debug!("No virtual environment found");
-                        (SitePackagesPaths::default(), None)
-                    })
-                } else {
-                    tracing::debug!("Resolving {origin}: {path}");
-                    PythonEnvironment::new(path, origin.clone(), system)?.into_settings(system)?
-                }
             }
 
-            PythonPath::KnownSitePackages(paths) => (
+            PythonEnvironmentPath::Explicit(prefix, origin) => {
+                tracing::debug!("Resolving {origin}: {prefix}");
+                PythonEnvironment::new(prefix, origin.clone(), system)?.into_settings(system)?
+            }
+
+            PythonEnvironmentPath::Testing(paths) => (
                 paths
                     .iter()
                     .map(|path| canonicalize(path, system))
@@ -307,6 +289,64 @@ impl SearchPaths {
         })
     }
 
+    fn discover_python_environment(
+        system: &dyn System,
+        project_root: &SystemPath,
+    ) -> Result<(SitePackagesPaths, Option<PythonVersionWithSource>), SitePackagesDiscoveryError>
+    {
+        fn resolve_environment(
+            system: &dyn System,
+            path: &SystemPath,
+            origin: SysPrefixPathOrigin,
+        ) -> Result<(SitePackagesPaths, Option<PythonVersionWithSource>), SitePackagesDiscoveryError>
+        {
+            tracing::debug!("Resolving {origin}: {path}");
+            PythonEnvironment::new(path, origin, system)?.into_settings(system)
+        }
+
+        if let Ok(virtual_env) = system.env_var("VIRTUAL_ENV") {
+            return resolve_environment(
+                system,
+                SystemPath::new(&virtual_env),
+                SysPrefixPathOrigin::VirtualEnvVar,
+            );
+        }
+
+        if let Ok(conda_env) = system.env_var("CONDA_PREFIX") {
+            return resolve_environment(
+                system,
+                SystemPath::new(&conda_env),
+                SysPrefixPathOrigin::CondaPrefixVar,
+            );
+        }
+
+        tracing::debug!("Discovering virtual environment in `{project_root}`");
+        let virtual_env_directory = project_root.join(".venv");
+
+        match PythonEnvironment::new(
+            &virtual_env_directory,
+            SysPrefixPathOrigin::LocalVenv,
+            system,
+        )
+        .and_then(|venv| venv.into_settings(system))
+        {
+            Ok(settings) => return Ok(settings),
+            Err(err) => {
+                if system.is_directory(&virtual_env_directory) {
+                    tracing::debug!(
+                        "Ignoring automatically detected virtual environment at `{}`: {}",
+                        &virtual_env_directory,
+                        err
+                    );
+                }
+            }
+        }
+
+        tracing::debug!("No virtual environment found");
+
+        Ok((SitePackagesPaths::default(), None))
+    }
+
     pub(crate) fn try_register_static_roots(&self, db: &dyn Db) {
         let files = db.files();
         for path in self.static_paths.iter().chain(self.site_packages.iter()) {
@@ -1494,7 +1534,7 @@ mod tests {
                 python_platform: PythonPlatform::default(),
                 search_paths: SearchPathSettings {
                     custom_typeshed: Some(custom_typeshed),
-                    python_path: PythonPath::KnownSitePackages(vec![site_packages]),
+                    python_environment: PythonEnvironmentPath::Testing(vec![site_packages]),
                     ..SearchPathSettings::new(vec![src.clone()])
                 }
                 .to_search_paths(db.system(), db.vendored())
@@ -2009,7 +2049,7 @@ not_a_directory
                 python_version: PythonVersionWithSource::default(),
                 python_platform: PythonPlatform::default(),
                 search_paths: SearchPathSettings {
-                    python_path: PythonPath::KnownSitePackages(vec![
+                    python_environment: PythonEnvironmentPath::Testing(vec![
                         venv_site_packages,
                         system_site_packages,
                     ]),
@@ -2126,7 +2166,7 @@ not_a_directory
                 python_version: PythonVersionWithSource::default(),
                 python_platform: PythonPlatform::default(),
                 search_paths: SearchPathSettings {
-                    python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
+                    python_environment: PythonEnvironmentPath::Testing(vec![site_packages.clone()]),
                     ..SearchPathSettings::new(vec![project_directory])
                 }
                 .to_search_paths(db.system(), db.vendored())

crates/ty_python_semantic/src/module_resolver/testing.rs

@@ -8,7 +8,8 @@ use ruff_python_ast::PythonVersion;
 use crate::db::tests::TestDb;
 use crate::program::{Program, SearchPathSettings};
 use crate::{
-    ProgramSettings, PythonPath, PythonPlatform, PythonVersionSource, PythonVersionWithSource,
+    ProgramSettings, PythonEnvironmentPath, PythonPlatform, PythonVersionSource,
+    PythonVersionWithSource,
 };
 
 /// A test case for the module resolver.
@@ -245,7 +246,7 @@ impl TestCaseBuilder<MockedTypeshed> {
                 python_platform,
                 search_paths: SearchPathSettings {
                     custom_typeshed: Some(typeshed.clone()),
-                    python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
+                    python_environment: PythonEnvironmentPath::Testing(vec![site_packages.clone()]),
                     ..SearchPathSettings::new(vec![src.clone()])
                 }
                 .to_search_paths(db.system(), db.vendored())
@@ -305,7 +306,7 @@ impl TestCaseBuilder<VendoredTypeshed> {
                 },
                 python_platform,
                 search_paths: SearchPathSettings {
-                    python_path: PythonPath::KnownSitePackages(vec![site_packages.clone()]),
+                    python_environment: PythonEnvironmentPath::Testing(vec![site_packages.clone()]),
                     ..SearchPathSettings::new(vec![src.clone()])
                 }
                 .to_search_paths(db.system(), db.vendored())

crates/ty_python_semantic/src/program.rs

@@ -1,9 +1,8 @@
 use std::sync::Arc;
 
-use crate::Db;
 use crate::module_resolver::{SearchPathValidationError, SearchPaths};
 use crate::python_platform::PythonPlatform;
-use crate::site_packages::SysPrefixPathOrigin;
+use crate::{Db, SysPrefixPathOrigin};
 
 use ruff_db::diagnostic::Span;
 use ruff_db::files::system_path_to_file;
@@ -174,9 +173,9 @@ pub struct SearchPathSettings {
     /// bundled as a zip file in the binary
     pub custom_typeshed: Option<SystemPathBuf>,
 
-    /// Path to the Python installation from which ty resolves third party dependencies
+    /// Path to the Python environment from which ty resolves third party dependencies
     /// and their type information.
-    pub python_path: PythonPath,
+    pub python_environment: PythonEnvironmentPath,
 }
 
 impl SearchPathSettings {
@@ -192,7 +191,7 @@ impl SearchPathSettings {
             src_roots: vec![],
             extra_paths: vec![],
             custom_typeshed: None,
-            python_path: PythonPath::KnownSitePackages(vec![]),
+            python_environment: PythonEnvironmentPath::Testing(vec![]),
         }
     }
 
@@ -206,8 +205,16 @@ impl SearchPathSettings {
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
-pub enum PythonPath {
+pub enum PythonEnvironmentPath {
-    /// A path that either represents the value of [`sys.prefix`] at runtime in Python
+    /// The path to the Python environment isn't known. Try to discover the Python environment
+    /// by inspecting environment variables, the project structure, etc. and derive the path from it.
+    ///
+    /// The path is the project root in which to search for a Python environment.
+    Discover(SystemPathBuf),
+
+    /// Path to a Python environment that is explicitly specified.
+    ///
+    /// The path that either represents the value of [`sys.prefix`] at runtime in Python
     /// for a given Python executable, or which represents a path relative to `sys.prefix`
     /// that we will attempt later to resolve into `sys.prefix`. Exactly which this variant
     /// represents depends on the [`SysPrefixPathOrigin`] element in the tuple.
@@ -222,17 +229,17 @@ pub enum PythonPath {
     /// `/opt/homebrew/lib/python3.X/site-packages`.
     ///
     /// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix
-    IntoSysPrefix(SystemPathBuf, SysPrefixPathOrigin),
+    Explicit(SystemPathBuf, SysPrefixPathOrigin),
 
-    /// Resolved site packages paths.
+    /// Don't search for a Python environment, instead use the provided site packages paths.
     ///
     /// This variant is mainly intended for testing where we want to skip resolving `site-packages`
     /// because it would unnecessarily complicate the test setup.
-    KnownSitePackages(Vec<SystemPathBuf>),
+    Testing(Vec<SystemPathBuf>),
 }
 
-impl PythonPath {
+impl PythonEnvironmentPath {
-    pub fn sys_prefix(path: impl Into<SystemPathBuf>, origin: SysPrefixPathOrigin) -> Self {
+    pub fn explicit(path: impl Into<SystemPathBuf>, origin: SysPrefixPathOrigin) -> Self {
-        Self::IntoSysPrefix(path.into(), origin)
+        Self::Explicit(path.into(), origin)
     }
 }

crates/ty_test/src/lib.rs

@@ -21,7 +21,7 @@ use std::fmt::Write;
 use ty_python_semantic::pull_types::pull_types;
 use ty_python_semantic::types::check_types;
 use ty_python_semantic::{
-    Program, ProgramSettings, PythonPath, PythonPlatform, PythonVersionSource,
+    Program, ProgramSettings, PythonEnvironmentPath, PythonPlatform, PythonVersionSource,
     PythonVersionWithSource, SearchPathSettings, SysPrefixPathOrigin,
 };
 
@@ -271,15 +271,15 @@ fn run_test(
             src_roots: vec![src_path],
             extra_paths: configuration.extra_paths().unwrap_or_default().to_vec(),
             custom_typeshed: custom_typeshed_path.map(SystemPath::to_path_buf),
-            python_path: configuration
+            python_environment: configuration
                 .python()
                 .map(|sys_prefix| {
-                    PythonPath::IntoSysPrefix(
+                    PythonEnvironmentPath::explicit(
                         sys_prefix.to_path_buf(),
                         SysPrefixPathOrigin::PythonCliFlag,
                     )
                 })
-                .unwrap_or(PythonPath::KnownSitePackages(vec![])),
+                .unwrap_or(PythonEnvironmentPath::Testing(vec![])),
         }
         .to_search_paths(db.system(), db.vendored())
         .expect("Failed to resolve search path settings"),