language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-10-02 14:52:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

[ty] Add basic support for non-virtual Python environments (#17991)

Browse source 
This adds basic support for non-virtual Python environments by accepting
a directory without a `pyvenv.cfg` which allows existing, subsequent
site-packages discovery logic to succeed. We can do better here in the
long-term, by adding more eager validation (for error messages) and
parsing the Python version from the discovered site-packages directory
(which isn't relevant yet, because we don't use the discovered Python
version from virtual environments as the default `--python-version` yet
either).

Related

- https://github.com/astral-sh/ty/issues/265
- https://github.com/astral-sh/ty/issues/193

You can review this commit by commit if it makes you happy.

I tested this manually; I think refactoring the test setup is going to
be a bit more invasive so I'll stack it on top (see
https://github.com/astral-sh/ruff/pull/17996).

```
❯ uv run ty check --python /Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none/ -vv example
2025-05-09 12:06:33.685911 DEBUG Version: 0.0.0-alpha.7 (f9c4c8999 2025-05-08)
2025-05-09 12:06:33.685987 DEBUG Architecture: aarch64, OS: macos, case-sensitive: case-insensitive
2025-05-09 12:06:33.686002 DEBUG Searching for a project in '/Users/zb/workspace/ty'
2025-05-09 12:06:33.686123 DEBUG Resolving requires-python constraint: `>=3.8`
2025-05-09 12:06:33.686129 DEBUG Resolved requires-python constraint to: 3.8
2025-05-09 12:06:33.686142 DEBUG Project without `tool.ty` section: '/Users/zb/workspace/ty'
2025-05-09 12:06:33.686147 DEBUG Searching for a user-level configuration at `/Users/zb/.config/ty/ty.toml`
2025-05-09 12:06:33.686156 INFO Defaulting to python-platform `darwin`
2025-05-09 12:06:33.68636 INFO Python version: Python 3.8, platform: darwin
2025-05-09 12:06:33.686375 DEBUG Adding first-party search path '/Users/zb/workspace/ty'
2025-05-09 12:06:33.68638 DEBUG Using vendored stdlib
2025-05-09 12:06:33.686634 DEBUG Discovering site-packages paths from sys-prefix `/Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none` (`--python` argument')
2025-05-09 12:06:33.686667 DEBUG Attempting to parse virtual environment metadata at '/Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none/pyvenv.cfg'
2025-05-09 12:06:33.686671 DEBUG Searching for site-packages directory in `sys.prefix` path `/Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none`
2025-05-09 12:06:33.686702 DEBUG Resolved site-packages directories for this environment are: ["/Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none/lib/python3.10/site-packages"]
2025-05-09 12:06:33.686706 DEBUG Adding site-packages search path '/Users/zb/.local/share/uv/python/cpython-3.10.17-macos-aarch64-none/lib/python3.10/site-packages'
...

❯ uv run ty check --python /tmp -vv example
2025-05-09 15:36:10.819416 DEBUG Version: 0.0.0-alpha.7 (f9c4c8999 2025-05-08)
2025-05-09 15:36:10.819708 DEBUG Architecture: aarch64, OS: macos, case-sensitive: case-insensitive
2025-05-09 15:36:10.820118 DEBUG Searching for a project in '/Users/zb/workspace/ty'
2025-05-09 15:36:10.821652 DEBUG Resolving requires-python constraint: `>=3.8`
2025-05-09 15:36:10.821667 DEBUG Resolved requires-python constraint to: 3.8
2025-05-09 15:36:10.8217 DEBUG Project without `tool.ty` section: '/Users/zb/workspace/ty'
2025-05-09 15:36:10.821888 DEBUG Searching for a user-level configuration at `/Users/zb/.config/ty/ty.toml`
2025-05-09 15:36:10.822072 INFO Defaulting to python-platform `darwin`
2025-05-09 15:36:10.822439 INFO Python version: Python 3.8, platform: darwin
2025-05-09 15:36:10.822773 DEBUG Adding first-party search path '/Users/zb/workspace/ty'
2025-05-09 15:36:10.822929 DEBUG Using vendored stdlib
2025-05-09 15:36:10.829872 DEBUG Discovering site-packages paths from sys-prefix `/tmp` (`--python` argument')
2025-05-09 15:36:10.829911 DEBUG Attempting to parse virtual environment metadata at '/private/tmp/pyvenv.cfg'
2025-05-09 15:36:10.829917 DEBUG Searching for site-packages directory in `sys.prefix` path `/private/tmp`
ty failed
  Cause: Invalid search path settings
  Cause: Failed to discover the site-packages directory: Failed to search the `lib` directory of the Python installation at `sys.prefix` path `/private/tmp` for `site-packages`
```
This commit is contained in:
Zanie Blue 2025-05-10 15:17:47 -05:00 • committed by GitHub
parent 5ecd560c6f
commit 316e406ca4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 123 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

8
crates/ty_python_semantic/src/module_resolver/resolver.rs
View file

@ -14,7 +14,7 @@ use ruff_python_ast::PythonVersion;
use crate::db::Db; use crate::db::Db;
use crate::module_name::ModuleName; use crate::module_name::ModuleName;
use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions}; use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions};
use crate::site_packages::{SitePackagesDiscoveryError, SysPrefixPathOrigin, VirtualEnvironment}; use crate::site_packages::{PythonEnvironment, SitePackagesDiscoveryError, SysPrefixPathOrigin};
use crate::{Program, PythonPath, SearchPathSettings}; use crate::{Program, PythonPath, SearchPathSettings};
use super::module::{Module, ModuleKind}; use super::module::{Module, ModuleKind};
@ -243,8 +243,8 @@ impl SearchPaths {
// than the one resolved in the program settings because it indicates // than the one resolved in the program settings because it indicates
// that the `target-version` is incorrectly configured or that the // that the `target-version` is incorrectly configured or that the
// venv is out of date. // venv is out of date.
VirtualEnvironment::new(sys_prefix, *origin, system) PythonEnvironment::new(sys_prefix, *origin, system)
.and_then(|venv| venv.site_packages_directories(system))? .and_then(|env| env.site_packages_directories(system))?
} }
PythonPath::Discover(root) => { PythonPath::Discover(root) => {
@ -262,7 +262,7 @@ impl SearchPaths {
vec![] vec![]
}; };
match VirtualEnvironment::new( match PythonEnvironment::new(
virtual_env_path.clone(), virtual_env_path.clone(),
SysPrefixPathOrigin::LocalVenv, SysPrefixPathOrigin::LocalVenv,
system, system,

160
crates/ty_python_semantic/src/site_packages.rs
View file

@ -19,6 +19,47 @@ use ruff_python_ast::PythonVersion;
type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>; type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
#[derive(Debug)]
pub(crate) enum PythonEnvironment {
Virtual(VirtualEnvironment),
System(SystemEnvironment),
}
impl PythonEnvironment {
pub(crate) fn new(
path: impl AsRef<SystemPath>,
origin: SysPrefixPathOrigin,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> {
let path = SysPrefixPath::new(path, origin, system)?;
// Attempt to inspect as a virtual environment first
// TODO(zanieb): Consider avoiding the clone here by checking for `pyvenv.cfg` ahead-of-time
match VirtualEnvironment::new(path.clone(), origin, system) {
Ok(venv) => Ok(Self::Virtual(venv)),
// If there's not a `pyvenv.cfg` marker, attempt to inspect as a system environment
//
Err(SitePackagesDiscoveryError::NoPyvenvCfgFile(_, _)) => {
Ok(Self::System(SystemEnvironment::new(path)))
}
Err(err) => Err(err),
}
}
/// Returns the `site-packages` directories for this Python environment.
///
/// See the documentation for [`site_packages_directory_from_sys_prefix`] for more details.
pub(crate) fn site_packages_directories(
&self,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
match self {
Self::Virtual(env) => env.site_packages_directories(system),
Self::System(env) => env.site_packages_directories(system),
}
}
}
/// Abstraction for a Python virtual environment. /// Abstraction for a Python virtual environment.
/// ///
/// Most of this information is derived from the virtual environment's `pyvenv.cfg` file. /// Most of this information is derived from the virtual environment's `pyvenv.cfg` file.
@ -26,7 +67,7 @@ type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
/// depends on the tool that was used to create the virtual environment. /// depends on the tool that was used to create the virtual environment.
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct VirtualEnvironment { pub(crate) struct VirtualEnvironment {
venv_path: SysPrefixPath, root_path: SysPrefixPath,
base_executable_home_path: PythonHomePath, base_executable_home_path: PythonHomePath,
include_system_site_packages: bool, include_system_site_packages: bool,
@ -43,15 +84,7 @@ pub(crate) struct VirtualEnvironment {
impl VirtualEnvironment { impl VirtualEnvironment {
pub(crate) fn new( pub(crate) fn new(
path: impl AsRef<SystemPath>, path: SysPrefixPath,
origin: SysPrefixPathOrigin,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> {
Self::new_impl(path.as_ref(), origin, system)
}
fn new_impl(
path: &SystemPath,
origin: SysPrefixPathOrigin, origin: SysPrefixPathOrigin,
system: &dyn System, system: &dyn System,
) -> SitePackagesDiscoveryResult<Self> { ) -> SitePackagesDiscoveryResult<Self> {
@ -59,8 +92,7 @@ impl VirtualEnvironment {
index.checked_add(1).and_then(NonZeroUsize::new).unwrap() index.checked_add(1).and_then(NonZeroUsize::new).unwrap()
} }
let venv_path = SysPrefixPath::new(path, origin, system)?; let pyvenv_cfg_path = path.join("pyvenv.cfg");
let pyvenv_cfg_path = venv_path.join("pyvenv.cfg");
tracing::debug!("Attempting to parse virtual environment metadata at '{pyvenv_cfg_path}'"); tracing::debug!("Attempting to parse virtual environment metadata at '{pyvenv_cfg_path}'");
let pyvenv_cfg = system let pyvenv_cfg = system
@ -150,7 +182,7 @@ impl VirtualEnvironment {
}); });
let metadata = Self { let metadata = Self {
venv_path, root_path: path,
base_executable_home_path, base_executable_home_path,
include_system_site_packages, include_system_site_packages,
version, version,
@ -162,20 +194,20 @@ impl VirtualEnvironment {
/// Return a list of `site-packages` directories that are available from this virtual environment /// Return a list of `site-packages` directories that are available from this virtual environment
/// ///
/// See the documentation for `site_packages_dir_from_sys_prefix` for more details. /// See the documentation for [`site_packages_directory_from_sys_prefix`] for more details.
pub(crate) fn site_packages_directories( pub(crate) fn site_packages_directories(
&self, &self,
system: &dyn System, system: &dyn System,
) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> { ) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
let VirtualEnvironment { let VirtualEnvironment {
venv_path, root_path,
base_executable_home_path, base_executable_home_path,
include_system_site_packages, include_system_site_packages,
version, version,
} = self; } = self;
let mut site_packages_directories = vec![site_packages_directory_from_sys_prefix( let mut site_packages_directories = vec![site_packages_directory_from_sys_prefix(
venv_path, *version, system, root_path, *version, system,
)?]; )?];
if *include_system_site_packages { if *include_system_site_packages {
@ -199,7 +231,7 @@ impl VirtualEnvironment {
"Failed to resolve `sys.prefix` of the system Python installation \ "Failed to resolve `sys.prefix` of the system Python installation \
from the `home` value in the `pyvenv.cfg` file at `{}`. \ from the `home` value in the `pyvenv.cfg` file at `{}`. \
System site-packages will not be used for module resolution.", System site-packages will not be used for module resolution.",
venv_path.join("pyvenv.cfg") root_path.join("pyvenv.cfg")
); );
} }
} }
@ -209,12 +241,49 @@ System site-packages will not be used for module resolution.",
} }
} }
/// A Python environment that is _not_ a virtual environment.
///
/// This environment may or may not be one that is managed by the operating system itself, e.g.,
/// this captures both Homebrew-installed Python versions and the bundled macOS Python installation.
#[derive(Debug)]
pub(crate) struct SystemEnvironment {
root_path: SysPrefixPath,
}
impl SystemEnvironment {
/// Create a new system environment from the given path.
///
/// At this time, there is no eager validation and this is infallible. Instead, validation
/// will occur in [`site_packages_directory_from_sys_prefix`] — which will fail if there is not
/// a Python environment at the given path.
pub(crate) fn new(path: SysPrefixPath) -> Self {
Self { root_path: path }
}
/// Return a list of `site-packages` directories that are available from this environment.
///
/// See the documentation for [`site_packages_directory_from_sys_prefix`] for more details.
pub(crate) fn site_packages_directories(
&self,
system: &dyn System,
) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
let SystemEnvironment { root_path } = self;
let site_packages_directories = vec![site_packages_directory_from_sys_prefix(
root_path, None, system,
)?];
tracing::debug!("Resolved site-packages directories for this environment are: {site_packages_directories:?}");
Ok(site_packages_directories)
}
}
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub(crate) enum SitePackagesDiscoveryError { pub(crate) enum SitePackagesDiscoveryError {
#[error("Invalid {1}: `{0}` could not be canonicalized")] #[error("Invalid {1}: `{0}` could not be canonicalized")]
VenvDirCanonicalizationError(SystemPathBuf, SysPrefixPathOrigin, #[source] io::Error), EnvDirCanonicalizationError(SystemPathBuf, SysPrefixPathOrigin, #[source] io::Error),
#[error("Invalid {1}: `{0}` does not point to a directory on disk")] #[error("Invalid {1}: `{0}` does not point to a directory on disk")]
VenvDirIsNotADirectory(SystemPathBuf, SysPrefixPathOrigin), EnvDirNotDirectory(SystemPathBuf, SysPrefixPathOrigin),
#[error("{0} points to a broken venv with no pyvenv.cfg file")] #[error("{0} points to a broken venv with no pyvenv.cfg file")]
NoPyvenvCfgFile(SysPrefixPathOrigin, #[source] io::Error), NoPyvenvCfgFile(SysPrefixPathOrigin, #[source] io::Error),
#[error("Failed to parse the pyvenv.cfg file at {0} because {1}")] #[error("Failed to parse the pyvenv.cfg file at {0} because {1}")]
@ -401,7 +470,7 @@ impl SysPrefixPath {
let canonicalized = system let canonicalized = system
.canonicalize_path(unvalidated_path) .canonicalize_path(unvalidated_path)
.map_err(|io_err| { .map_err(|io_err| {
SitePackagesDiscoveryError::VenvDirCanonicalizationError( SitePackagesDiscoveryError::EnvDirCanonicalizationError(
unvalidated_path.to_path_buf(), unvalidated_path.to_path_buf(),
origin, origin,
io_err, io_err,
@ -414,7 +483,7 @@ impl SysPrefixPath {
origin, origin,
}) })
.ok_or_else(|| { .ok_or_else(|| {
SitePackagesDiscoveryError::VenvDirIsNotADirectory( SitePackagesDiscoveryError::EnvDirNotDirectory(
unvalidated_path.to_path_buf(), unvalidated_path.to_path_buf(),
origin, origin,
) )
@ -628,15 +697,19 @@ mod tests {
fn test(self) { fn test(self) {
let venv_path = self.build_mock_venv(); let venv_path = self.build_mock_venv();
let venv = VirtualEnvironment::new( let env = PythonEnvironment::new(
venv_path.clone(), venv_path.clone(),
SysPrefixPathOrigin::VirtualEnvVar, SysPrefixPathOrigin::VirtualEnvVar,
&self.system, &self.system,
) )
.unwrap(); .unwrap();
let PythonEnvironment::Virtual(venv) = &env else {
panic!("Expected a virtual environment; got {env:?}");
};
assert_eq!( assert_eq!(
venv.venv_path, venv.root_path,
SysPrefixPath { SysPrefixPath {
inner: self.system.canonicalize_path(&venv_path).unwrap(), inner: self.system.canonicalize_path(&venv_path).unwrap(),
origin: SysPrefixPathOrigin::VirtualEnvVar, origin: SysPrefixPathOrigin::VirtualEnvVar,
@ -663,7 +736,7 @@ mod tests {
}; };
assert_eq!(venv.base_executable_home_path, expected_home); assert_eq!(venv.base_executable_home_path, expected_home);
let site_packages_directories = venv.site_packages_directories(&self.system).unwrap(); let site_packages_directories = env.site_packages_directories(&self.system).unwrap();
let expected_venv_site_packages = if cfg!(target_os = "windows") { let expected_venv_site_packages = if cfg!(target_os = "windows") {
SystemPathBuf::from(r"\.venv\Lib\site-packages") SystemPathBuf::from(r"\.venv\Lib\site-packages")
} else if self.free_threaded { } else if self.free_threaded {
@ -782,8 +855,8 @@ mod tests {
fn reject_venv_that_does_not_exist() { fn reject_venv_that_does_not_exist() {
let system = TestSystem::default(); let system = TestSystem::default();
assert!(matches!( assert!(matches!(
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system), PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system),
Err(SitePackagesDiscoveryError::VenvDirCanonicalizationError(..)) Err(SitePackagesDiscoveryError::EnvDirCanonicalizationError(..))
)); ));
} }
@ -795,25 +868,30 @@ mod tests {
.write_file_all("/.venv", "") .write_file_all("/.venv", "")
.unwrap(); .unwrap();
assert!(matches!( assert!(matches!(
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system), PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system),
Err(SitePackagesDiscoveryError::VenvDirIsNotADirectory(..)) Err(SitePackagesDiscoveryError::EnvDirNotDirectory(..))
)); ));
} }
#[test] #[test]
fn reject_venv_with_no_pyvenv_cfg_file() { fn env_with_no_pyvenv_cfg_file() {
let system = TestSystem::default(); let system = TestSystem::default();
system system
.memory_file_system() .memory_file_system()
.create_directory_all("/.venv") .create_directory_all("/.venv")
.unwrap(); .unwrap();
assert!(matches!( let env =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system), PythonEnvironment::new("/.venv", SysPrefixPathOrigin::PythonCliFlag, &system).unwrap();
Err(SitePackagesDiscoveryError::NoPyvenvCfgFile( let PythonEnvironment::System(env) = env else {
SysPrefixPathOrigin::VirtualEnvVar, panic!("Expected a system environment; got {env:?}");
_ };
)) assert!(
)); env.root_path
== SysPrefixPath {
inner: system.canonicalize_path(SystemPath::new("/.venv")).unwrap(),
origin: SysPrefixPathOrigin::PythonCliFlag,
}
);
} }
#[test] #[test]
@ -825,7 +903,7 @@ mod tests {
.write_file_all(&pyvenv_cfg_path, "home = bar = /.venv/bin") .write_file_all(&pyvenv_cfg_path, "home = bar = /.venv/bin")
.unwrap(); .unwrap();
let venv_result = let venv_result =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system); PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system);
assert!(matches!( assert!(matches!(
venv_result, venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError( Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
@ -845,7 +923,7 @@ mod tests {
.write_file_all(&pyvenv_cfg_path, "home =") .write_file_all(&pyvenv_cfg_path, "home =")
.unwrap(); .unwrap();
let venv_result = let venv_result =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system); PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system);
assert!(matches!( assert!(matches!(
venv_result, venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError( Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
@ -865,7 +943,7 @@ mod tests {
.write_file_all(&pyvenv_cfg_path, "= whatever") .write_file_all(&pyvenv_cfg_path, "= whatever")
.unwrap(); .unwrap();
let venv_result = let venv_result =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system); PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system);
assert!(matches!( assert!(matches!(
venv_result, venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError( Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
@ -883,7 +961,7 @@ mod tests {
let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
memory_fs.write_file_all(&pyvenv_cfg_path, "").unwrap(); memory_fs.write_file_all(&pyvenv_cfg_path, "").unwrap();
let venv_result = let venv_result =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system); PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system);
assert!(matches!( assert!(matches!(
venv_result, venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError( Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
@ -903,7 +981,7 @@ mod tests {
.write_file_all(&pyvenv_cfg_path, "home = foo") .write_file_all(&pyvenv_cfg_path, "home = foo")
.unwrap(); .unwrap();
let venv_result = let venv_result =
VirtualEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system); PythonEnvironment::new("/.venv", SysPrefixPathOrigin::VirtualEnvVar, &system);
assert!(matches!( assert!(matches!(
venv_result, venv_result,
Err(SitePackagesDiscoveryError::PyvenvCfgParseError( Err(SitePackagesDiscoveryError::PyvenvCfgParseError(