[ty] Infer the Python version from --python=<system installation> on Unix (#18550)

crates/ty_python_semantic/src/module_resolver/resolver.rs

@@ -1,8 +1,9 @@
 use std::borrow::Cow;
 use std::fmt;
 use std::iter::FusedIterator;
-use std::str::Split;
+use std::str::{FromStr, Split};
 
+use camino::Utf8Component;
 use compact_str::format_compact;
 use rustc_hash::{FxBuildHasher, FxHashSet};
 
@@ -15,7 +16,9 @@ use crate::db::Db;
 use crate::module_name::ModuleName;
 use crate::module_resolver::typeshed::{TypeshedVersions, vendored_typeshed_versions};
 use crate::site_packages::{PythonEnvironment, SitePackagesPaths, SysPrefixPathOrigin};
-use crate::{Program, PythonPath, PythonVersionWithSource, SearchPathSettings};
+use crate::{
+    Program, PythonPath, PythonVersionSource, PythonVersionWithSource, SearchPathSettings,
+};
 
 use super::module::{Module, ModuleKind};
 use super::path::{ModulePath, SearchPath, SearchPathValidationError};
@@ -155,10 +158,12 @@ pub struct SearchPaths {
 
     typeshed_versions: TypeshedVersions,
 
-    /// The Python version for the search paths, if any.
+    /// The Python version implied by the virtual environment.
     ///
-    /// This is read from the `pyvenv.cfg` if present.
-    python_version: Option<PythonVersionWithSource>,
+    /// If this environment was a system installation or the `pyvenv.cfg` file
+    /// of the virtual environment did not contain a `version` or `version_info` key,
+    /// this field will be `None`.
+    python_version_from_pyvenv_cfg: Option<PythonVersionWithSource>,
 }
 
 impl SearchPaths {
@@ -304,7 +309,7 @@ impl SearchPaths {
             static_paths,
             site_packages,
             typeshed_versions,
-            python_version,
+            python_version_from_pyvenv_cfg: python_version,
         })
     }
 
@@ -330,8 +335,50 @@ impl SearchPaths {
         &self.typeshed_versions
     }
 
-    pub fn python_version(&self) -> Option<&PythonVersionWithSource> {
-        self.python_version.as_ref()
+    pub fn try_resolve_installation_python_version(&self) -> Option<Cow<PythonVersionWithSource>> {
+        if let Some(version) = self.python_version_from_pyvenv_cfg.as_ref() {
+            return Some(Cow::Borrowed(version));
+        }
+
+        if cfg!(windows) {
+            // The path to `site-packages` on Unix is
+            // `<sys.prefix>/lib/pythonX.Y/site-packages`,
+            // but on Windows it's `<sys.prefix>/Lib/site-packages`.
+            return None;
+        }
+
+        let primary_site_packages = self.site_packages.first()?.as_system_path()?;
+
+        let mut site_packages_ancestor_components =
+            primary_site_packages.components().rev().skip(1).map(|c| {
+                // This should have all been validated in `site_packages.rs`
+                // when we resolved the search paths for the project.
+                debug_assert!(
+                    matches!(c, Utf8Component::Normal(_)),
+                    "Unexpected component in site-packages path `{c:?}` \
+                    (expected `site-packages` to be an absolute path with symlinks resolved, \
+                    located at `<sys.prefix>/lib/pythonX.Y/site-packages`)"
+                );
+
+                c.as_str()
+            });
+
+        let parent_component = site_packages_ancestor_components.next()?;
+
+        if site_packages_ancestor_components.next()? != "lib" {
+            return None;
+        }
+
+        let version = parent_component
+            .strip_prefix("python")
+            .or_else(|| parent_component.strip_prefix("pypy"))?
+            .trim_end_matches('t');
+
+        let version = PythonVersion::from_str(version).ok()?;
+        let source = PythonVersionSource::InstallationDirectoryLayout {
+            site_packages_parent_dir: Box::from(parent_component),
+        };
+        Some(Cow::Owned(PythonVersionWithSource { version, source }))
     }
 }
 
@@ -351,7 +398,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
         static_paths,
         site_packages,
         typeshed_versions: _,
-        python_version: _,
+        python_version_from_pyvenv_cfg: _,
     } = Program::get(db).search_paths(db);
 
     let mut dynamic_paths = Vec::new();

crates/ty_python_semantic/src/module_resolver/typeshed.rs

@@ -4,7 +4,7 @@ use std::num::{NonZeroU16, NonZeroUsize};
 use std::ops::{RangeFrom, RangeInclusive};
 use std::str::FromStr;
 
-use ruff_python_ast::PythonVersion;
+use ruff_python_ast::{PythonVersion, PythonVersionDeserializationError};
 use rustc_hash::FxHashMap;
 
 use crate::Program;
@@ -49,55 +49,26 @@ impl fmt::Display for TypeshedVersionsParseError {
 
 impl std::error::Error for TypeshedVersionsParseError {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        if let TypeshedVersionsParseErrorKind::IntegerParsingFailure { err, .. } = &self.reason {
-            Some(err)
+        if let TypeshedVersionsParseErrorKind::VersionParseError(err) = &self.reason {
+            err.source()
         } else {
             None
         }
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone, thiserror::Error)]
 pub(crate) enum TypeshedVersionsParseErrorKind {
+    #[error("File has too many lines ({0}); maximum allowed is {max_allowed}", max_allowed = NonZeroU16::MAX)]
     TooManyLines(NonZeroUsize),
+    #[error("Expected every non-comment line to have exactly one colon")]
     UnexpectedNumberOfColons,
+    #[error("Expected all components of '{0}' to be valid Python identifiers")]
     InvalidModuleName(String),
+    #[error("Expected every non-comment line to have exactly one '-' character")]
     UnexpectedNumberOfHyphens,
-    UnexpectedNumberOfPeriods(String),
-    IntegerParsingFailure {
-        version: String,
-        err: std::num::ParseIntError,
-    },
-}
-
-impl fmt::Display for TypeshedVersionsParseErrorKind {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::TooManyLines(num_lines) => write!(
-                f,
-                "File has too many lines ({num_lines}); maximum allowed is {}",
-                NonZeroU16::MAX
-            ),
-            Self::UnexpectedNumberOfColons => {
-                f.write_str("Expected every non-comment line to have exactly one colon")
-            }
-            Self::InvalidModuleName(name) => write!(
-                f,
-                "Expected all components of '{name}' to be valid Python identifiers"
-            ),
-            Self::UnexpectedNumberOfHyphens => {
-                f.write_str("Expected every non-comment line to have exactly one '-' character")
-            }
-            Self::UnexpectedNumberOfPeriods(format) => write!(
-                f,
-                "Expected all versions to be in the form {{MAJOR}}.{{MINOR}}; got '{format}'"
-            ),
-            Self::IntegerParsingFailure { version, err } => write!(
-                f,
-                "Failed to convert '{version}' to a pair of integers due to {err}",
-            ),
-        }
-    }
+    #[error("{0}")]
+    VersionParseError(#[from] PythonVersionDeserializationError),
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -304,12 +275,12 @@ impl FromStr for PyVersionRange {
         let mut parts = s.split('-').map(str::trim);
         match (parts.next(), parts.next(), parts.next()) {
             (Some(lower), Some(""), None) => {
-                let lower = python_version_from_versions_file_string(lower)?;
+                let lower = PythonVersion::from_str(lower)?;
                 Ok(Self::AvailableFrom(lower..))
             }
             (Some(lower), Some(upper), None) => {
-                let lower = python_version_from_versions_file_string(lower)?;
-                let upper = python_version_from_versions_file_string(upper)?;
+                let lower = PythonVersion::from_str(lower)?;
+                let upper = PythonVersion::from_str(upper)?;
                 Ok(Self::AvailableWithin(lower..=upper))
             }
             _ => Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfHyphens),
@@ -328,23 +299,6 @@ impl fmt::Display for PyVersionRange {
     }
 }
 
-fn python_version_from_versions_file_string(
-    s: &str,
-) -> Result<PythonVersion, TypeshedVersionsParseErrorKind> {
-    let mut parts = s.split('.').map(str::trim);
-    let (Some(major), Some(minor), None) = (parts.next(), parts.next(), parts.next()) else {
-        return Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods(
-            s.to_string(),
-        ));
-    };
-    PythonVersion::try_from((major, minor)).map_err(|int_parse_error| {
-        TypeshedVersionsParseErrorKind::IntegerParsingFailure {
-            version: s.to_string(),
-            err: int_parse_error,
-        }
-    })
-}
-
 #[cfg(test)]
 mod tests {
     use std::fmt::Write as _;
@@ -681,15 +635,17 @@ foo: 3.8-   # trailing comment
             TypeshedVersions::from_str("foo: 38-"),
             Err(TypeshedVersionsParseError {
                 line_number: ONE,
-                reason: TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods("38".to_string())
+                reason: TypeshedVersionsParseErrorKind::VersionParseError(
+                    PythonVersionDeserializationError::WrongPeriodNumber(Box::from("38"))
+                )
             })
         );
         assert_eq!(
             TypeshedVersions::from_str("foo: 3..8-"),
             Err(TypeshedVersionsParseError {
                 line_number: ONE,
-                reason: TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods(
-                    "3..8".to_string()
+                reason: TypeshedVersionsParseErrorKind::VersionParseError(
+                    PythonVersionDeserializationError::WrongPeriodNumber(Box::from("3..8"))
                 )
             })
         );
@@ -697,8 +653,8 @@ foo: 3.8-   # trailing comment
             TypeshedVersions::from_str("foo: 3.8-3..11"),
             Err(TypeshedVersionsParseError {
                 line_number: ONE,
-                reason: TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods(
-                    "3..11".to_string()
+                reason: TypeshedVersionsParseErrorKind::VersionParseError(
+                    PythonVersionDeserializationError::WrongPeriodNumber(Box::from("3..11"))
                 )
             })
         );
@@ -708,20 +664,30 @@ foo: 3.8-   # trailing comment
     fn invalid_typeshed_versions_non_digits() {
         let err = TypeshedVersions::from_str("foo: 1.two-").unwrap_err();
         assert_eq!(err.line_number, ONE);
-        let TypeshedVersionsParseErrorKind::IntegerParsingFailure { version, err } = err.reason
+        let TypeshedVersionsParseErrorKind::VersionParseError(
+            PythonVersionDeserializationError::InvalidMinorVersion(invalid_minor, parse_error),
+        ) = err.reason
         else {
-            panic!()
+            panic!(
+                "Expected an invalid-minor-version parse error, got `{}`",
+                err.reason
+            )
         };
-        assert_eq!(version, "1.two".to_string());
-        assert_eq!(*err.kind(), IntErrorKind::InvalidDigit);
+        assert_eq!(&*invalid_minor, "two");
+        assert_eq!(*parse_error.kind(), IntErrorKind::InvalidDigit);
 
         let err = TypeshedVersions::from_str("foo: 3.8-four.9").unwrap_err();
         assert_eq!(err.line_number, ONE);
-        let TypeshedVersionsParseErrorKind::IntegerParsingFailure { version, err } = err.reason
+        let TypeshedVersionsParseErrorKind::VersionParseError(
+            PythonVersionDeserializationError::InvalidMajorVersion(invalid_major, parse_error),
+        ) = err.reason
         else {
-            panic!()
+            panic!(
+                "Expected an invalid-major-version parse error, got `{}`",
+                err.reason
+            )
         };
-        assert_eq!(version, "four.9".to_string());
-        assert_eq!(*err.kind(), IntErrorKind::InvalidDigit);
+        assert_eq!(&*invalid_major, "four");
+        assert_eq!(*parse_error.kind(), IntErrorKind::InvalidDigit);
     }
 }

crates/ty_python_semantic/src/program.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use crate::Db;
@@ -38,7 +39,7 @@ impl Program {
             .with_context(|| "Invalid search path settings")?;
 
         let python_version_with_source =
-            Self::resolve_python_version(python_version_with_source, search_paths.python_version());
+            Self::resolve_python_version(python_version_with_source, &search_paths);
 
         tracing::info!(
             "Python version: Python {python_version}, platform: {python_platform}",
@@ -58,10 +59,14 @@ impl Program {
 
     fn resolve_python_version(
         config_value: Option<PythonVersionWithSource>,
-        environment_value: Option<&PythonVersionWithSource>,
+        search_paths: &SearchPaths,
     ) -> PythonVersionWithSource {
         config_value
-            .or_else(|| environment_value.cloned())
+            .or_else(|| {
+                search_paths
+                    .try_resolve_installation_python_version()
+                    .map(Cow::into_owned)
+            })
             .unwrap_or_default()
     }
 
@@ -79,7 +84,7 @@ impl Program {
         let search_paths = SearchPaths::from_settings(db, &search_paths)?;
 
         let new_python_version =
-            Self::resolve_python_version(python_version_with_source, search_paths.python_version());
+            Self::resolve_python_version(python_version_with_source, &search_paths);
 
         if self.search_paths(db) != &search_paths {
             tracing::debug!("Updating search paths");
@@ -112,8 +117,11 @@ impl Program {
         let search_paths = SearchPaths::from_settings(db, search_path_settings)?;
 
         let current_python_version = self.python_version_with_source(db);
-        let python_version_from_environment =
-            search_paths.python_version().cloned().unwrap_or_default();
+
+        let python_version_from_environment = search_paths
+            .try_resolve_installation_python_version()
+            .map(Cow::into_owned)
+            .unwrap_or_default();
 
         if current_python_version != &python_version_from_environment
             && current_python_version.source.priority()
@@ -153,6 +161,13 @@ pub enum PythonVersionSource {
     /// The virtual environment might have been configured, activated or inferred.
     PyvenvCfgFile(PythonVersionFileSource),
 
+    /// Value inferred from the layout of the Python installation.
+    ///
+    /// This only ever applies on Unix. On Unix, the `site-packages` directory
+    /// will always be at `sys.prefix/lib/pythonX.Y/site-packages`,
+    /// so we can infer the Python version from the parent directory of `site-packages`.
+    InstallationDirectoryLayout { site_packages_parent_dir: Box<str> },
+
     /// The value comes from a CLI argument, while it's left open if specified using a short argument,
     /// long argument (`--extra-paths`) or `--config key=value`.
     Cli,
@@ -169,22 +184,26 @@ impl PythonVersionSource {
             PythonVersionSource::PyvenvCfgFile(_) => PythonSourcePriority::PyvenvCfgFile,
             PythonVersionSource::ConfigFile(_) => PythonSourcePriority::ConfigFile,
             PythonVersionSource::Cli => PythonSourcePriority::Cli,
+            PythonVersionSource::InstallationDirectoryLayout { .. } => {
+                PythonSourcePriority::InstallationDirectoryLayout
+            }
         }
     }
 }
 
 /// The priority in which Python version sources are considered.
-/// A higher value means a higher priority.
+/// The lower down the variant appears in this enum, the higher its priority.
 ///
 /// For example, if a Python version is specified in a pyproject.toml file
 /// but *also* via a CLI argument, the CLI argument will take precedence.
 #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)]
 #[cfg_attr(test, derive(strum_macros::EnumIter))]
 enum PythonSourcePriority {
-    Default = 0,
-    PyvenvCfgFile = 1,
-    ConfigFile = 2,
-    Cli = 3,
+    Default,
+    InstallationDirectoryLayout,
+    PyvenvCfgFile,
+    ConfigFile,
+    Cli,
 }
 
 /// Information regarding the file and [`TextRange`] of the configuration
@@ -312,7 +331,9 @@ mod tests {
                         match other {
                             PythonSourcePriority::Cli => assert!(other > priority, "{other:?}"),
                             PythonSourcePriority::ConfigFile => assert_eq!(priority, other),
-                            PythonSourcePriority::PyvenvCfgFile | PythonSourcePriority::Default => {
+                            PythonSourcePriority::PyvenvCfgFile
+                            | PythonSourcePriority::Default
+                            | PythonSourcePriority::InstallationDirectoryLayout => {
                                 assert!(priority > other, "{other:?}");
                             }
                         }
@@ -327,6 +348,24 @@ mod tests {
                                 assert!(other > priority, "{other:?}");
                             }
                             PythonSourcePriority::PyvenvCfgFile => assert_eq!(priority, other),
+                            PythonSourcePriority::Default
+                            | PythonSourcePriority::InstallationDirectoryLayout => {
+                                assert!(priority > other, "{other:?}");
+                            }
+                        }
+                    }
+                }
+                PythonSourcePriority::InstallationDirectoryLayout => {
+                    for other in PythonSourcePriority::iter() {
+                        match other {
+                            PythonSourcePriority::Cli
+                            | PythonSourcePriority::ConfigFile
+                            | PythonSourcePriority::PyvenvCfgFile => {
+                                assert!(other > priority, "{other:?}");
+                            }
+                            PythonSourcePriority::InstallationDirectoryLayout => {
+                                assert_eq!(priority, other);
+                            }
                             PythonSourcePriority::Default => assert!(priority > other, "{other:?}"),
                         }
                     }

crates/ty_python_semantic/src/util/diagnostics.rs

@@ -61,6 +61,23 @@ pub fn add_inferred_python_version_hint_to_diagnostic(
                 or in a configuration file",
             );
         }
+        crate::PythonVersionSource::InstallationDirectoryLayout {
+            site_packages_parent_dir,
+        } => {
+            // TODO: it would also be nice to tell them how we resolved this Python installation...
+            diagnostic.info(format_args!(
+                "Python {version} was assumed when {action} \
+                because of the layout of your Python installation"
+            ));
+            diagnostic.info(format_args!(
+                "The primary `site-packages` directory of your installation was found \
+                at `lib/{site_packages_parent_dir}/site-packages/`"
+            ));
+            diagnostic.info(
+                "No Python version was specified on the command line \
+                or in a configuration file",
+            );
+        }
         crate::PythonVersionSource::Default => {
             diagnostic.info(format_args!(
                 "Python {version} was assumed when {action} \