Change "toolchain" to "python" (#4735)

Whew this is a lot.

The user-facing changes are:

- `uv toolchain` to `uv python` e.g. `uv python find`, `uv python
install`, ...
- `UV_TOOLCHAIN_DIR` to` UV_PYTHON_INSTALL_DIR`
- `<UV_STATE_DIR>/toolchains` to `<UV_STATE_DIR>/python` (with
[automatic
migration](https://github.com/astral-sh/uv/pull/4735/files#r1663029330))
- User-facing messages no longer refer to toolchains, instead using
"Python", "Python versions" or "Python installations"

The internal changes are:

- `uv-toolchain` crate to `uv-python`
- `Toolchain` no longer referenced in type names
- Dropped unused `SystemPython` type (previously replaced)
- Clarified the type names for "managed Python installations"
- (more little things)
This commit is contained in:
Zanie Blue 2024-07-03 08:44:29 -04:00 committed by GitHub
parent 60fd98a5e4
commit dd7da6af5f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
105 changed files with 2629 additions and 2603 deletions

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,32 @@
// DO NOT EDIT
//
// Generated with `{{generated_with}}`
// From template at `{{generated_from}}`
pub(crate) const PYTHON_DOWNLOADS: &[ManagedPythonDownload] = &[
{{#versions}}
ManagedPythonDownload {
key: PythonInstallationKey {
major: {{value.major}},
minor: {{value.minor}},
patch: {{value.patch}},
implementation: LenientImplementationName::Known(ImplementationName::{{value.name}}),
arch: Arch(target_lexicon::Architecture::{{value.arch}}),
os: Os(target_lexicon::OperatingSystem::{{value.os}}),
{{#value.libc}}
libc: Libc::Some(target_lexicon::Environment::{{.}}),
{{/value.libc}}
{{^value.libc}}
libc: Libc::None,
{{/value.libc}}
},
url: "{{value.url}}",
{{#value.sha256}}
sha256: Some("{{.}}")
{{/value.sha256}}
{{^value.sha256}}
sha256: None
{{/value.sha256}}
},
{{/versions}}
];

View file

@ -0,0 +1,475 @@
use std::fmt::Display;
use std::io;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use crate::implementation::{
Error as ImplementationError, ImplementationName, LenientImplementationName,
};
use crate::installation::PythonInstallationKey;
use crate::platform::{self, Arch, Libc, Os};
use crate::{Interpreter, PythonRequest, PythonVersion, VersionRequest};
use thiserror::Error;
use uv_client::WrappedReqwestError;
use futures::TryStreamExt;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{debug, instrument};
use url::Url;
use uv_fs::{rename_with_retry, Simplified};
#[derive(Error, Debug)]
pub enum Error {
#[error(transparent)]
IO(#[from] io::Error),
#[error(transparent)]
ImplementationError(#[from] ImplementationError),
#[error("Invalid python version: {0}")]
InvalidPythonVersion(String),
#[error("Invalid request key, too many parts: {0}")]
TooManyParts(String),
#[error("Download failed")]
NetworkError(#[from] WrappedReqwestError),
#[error("Download failed")]
NetworkMiddlewareError(#[source] anyhow::Error),
#[error("Failed to extract archive: {0}")]
ExtractError(String, #[source] uv_extract::Error),
#[error("Invalid download url")]
InvalidUrl(#[from] url::ParseError),
#[error("Failed to create download directory")]
DownloadDirError(#[source] io::Error),
#[error("Failed to copy to: {0}", to.user_display())]
CopyError {
to: PathBuf,
#[source]
err: io::Error,
},
#[error("Failed to read managed Python installation directory: {0}", dir.user_display())]
ReadError {
dir: PathBuf,
#[source]
err: io::Error,
},
#[error("Failed to parse managed Python directory name: {0}")]
NameError(String),
#[error("Failed to parse request part")]
InvalidRequestPlatform(#[from] platform::Error),
#[error("Cannot download managed Python for request: {0}")]
InvalidRequestKind(PythonRequest),
// TODO(zanieb): Implement display for `PythonDownloadRequest`
#[error("No download found for request: {0:?}")]
NoDownloadFound(PythonDownloadRequest),
}
#[derive(Debug, PartialEq)]
pub struct ManagedPythonDownload {
key: PythonInstallationKey,
url: &'static str,
sha256: Option<&'static str>,
}
#[derive(Debug, Clone, Default, Eq, PartialEq)]
pub struct PythonDownloadRequest {
version: Option<VersionRequest>,
implementation: Option<ImplementationName>,
arch: Option<Arch>,
os: Option<Os>,
libc: Option<Libc>,
}
impl PythonDownloadRequest {
pub fn new(
version: Option<VersionRequest>,
implementation: Option<ImplementationName>,
arch: Option<Arch>,
os: Option<Os>,
libc: Option<Libc>,
) -> Self {
Self {
version,
implementation,
arch,
os,
libc,
}
}
#[must_use]
pub fn with_implementation(mut self, implementation: ImplementationName) -> Self {
self.implementation = Some(implementation);
self
}
#[must_use]
pub fn with_version(mut self, version: VersionRequest) -> Self {
self.version = Some(version);
self
}
#[must_use]
pub fn with_arch(mut self, arch: Arch) -> Self {
self.arch = Some(arch);
self
}
#[must_use]
pub fn with_os(mut self, os: Os) -> Self {
self.os = Some(os);
self
}
#[must_use]
pub fn with_libc(mut self, libc: Libc) -> Self {
self.libc = Some(libc);
self
}
/// Construct a new [`PythonDownloadRequest`] from a [`PythonRequest`] if possible.
///
/// Returns [`None`] if the request kind is not compatible with a download, e.g., it is
/// a request for a specific directory or executable name.
pub fn try_from_request(request: &PythonRequest) -> Option<Self> {
Self::from_request(request).ok()
}
/// Construct a new [`PythonDownloadRequest`] from a [`PythonRequest`].
pub fn from_request(request: &PythonRequest) -> Result<Self, Error> {
match request {
PythonRequest::Version(version) => Ok(Self::default().with_version(version.clone())),
PythonRequest::Implementation(implementation) => {
Ok(Self::default().with_implementation(*implementation))
}
PythonRequest::ImplementationVersion(implementation, version) => Ok(Self::default()
.with_implementation(*implementation)
.with_version(version.clone())),
PythonRequest::Key(request) => Ok(request.clone()),
PythonRequest::Any => Ok(Self::default()),
// We can't download a managed installation for these request kinds
PythonRequest::Directory(_)
| PythonRequest::ExecutableName(_)
| PythonRequest::File(_) => Err(Error::InvalidRequestKind(request.clone())),
}
}
/// Fill empty entries with default values.
///
/// Platform information is pulled from the environment.
#[must_use]
pub fn fill(mut self) -> Self {
if self.implementation.is_none() {
self.implementation = Some(ImplementationName::CPython);
}
if self.arch.is_none() {
self.arch = Some(Arch::from_env());
}
if self.os.is_none() {
self.os = Some(Os::from_env());
}
if self.libc.is_none() {
self.libc = Some(Libc::from_env());
}
self
}
/// Construct a new [`PythonDownloadRequest`] with platform information from the environment.
pub fn from_env() -> Result<Self, Error> {
Ok(Self::new(
None,
None,
Some(Arch::from_env()),
Some(Os::from_env()),
Some(Libc::from_env()),
))
}
pub fn implementation(&self) -> Option<&ImplementationName> {
self.implementation.as_ref()
}
pub fn version(&self) -> Option<&VersionRequest> {
self.version.as_ref()
}
pub fn arch(&self) -> Option<&Arch> {
self.arch.as_ref()
}
pub fn os(&self) -> Option<&Os> {
self.os.as_ref()
}
pub fn libc(&self) -> Option<&Libc> {
self.libc.as_ref()
}
/// Iterate over all [`PythonDownload`]'s that match this request.
pub fn iter_downloads(&self) -> impl Iterator<Item = &'static ManagedPythonDownload> + '_ {
ManagedPythonDownload::iter_all()
.filter(move |download| self.satisfied_by_download(download))
}
pub fn satisfied_by_key(&self, key: &PythonInstallationKey) -> bool {
if let Some(arch) = &self.arch {
if key.arch != *arch {
return false;
}
}
if let Some(os) = &self.os {
if key.os != *os {
return false;
}
}
if let Some(implementation) = &self.implementation {
if key.implementation != LenientImplementationName::from(*implementation) {
return false;
}
}
if let Some(version) = &self.version {
if !version.matches_major_minor_patch(key.major, key.minor, key.patch) {
return false;
}
}
true
}
pub fn satisfied_by_download(&self, download: &ManagedPythonDownload) -> bool {
self.satisfied_by_key(download.key())
}
pub fn satisfied_by_interpreter(&self, interpreter: &Interpreter) -> bool {
if let Some(version) = self.version() {
if !version.matches_interpreter(interpreter) {
return false;
}
}
if let Some(os) = self.os() {
if &Os::from(interpreter.platform().os()) != os {
return false;
}
}
if let Some(arch) = self.arch() {
if &Arch::from(&interpreter.platform().arch()) != arch {
return false;
}
}
if let Some(implementation) = self.implementation() {
if LenientImplementationName::from(interpreter.implementation_name())
!= LenientImplementationName::from(*implementation)
{
return false;
}
}
if let Some(libc) = self.libc() {
if &Libc::from(interpreter.platform().os()) != libc {
return false;
}
}
true
}
}
impl Display for PythonDownloadRequest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut parts = Vec::new();
if let Some(implementation) = self.implementation {
parts.push(implementation.to_string());
} else {
parts.push("any".to_string());
}
if let Some(version) = &self.version {
parts.push(version.to_string());
} else {
parts.push("any".to_string());
}
if let Some(os) = &self.os {
parts.push(os.to_string());
} else {
parts.push("any".to_string());
}
if let Some(arch) = self.arch {
parts.push(arch.to_string());
} else {
parts.push("any".to_string());
}
if let Some(libc) = self.libc {
parts.push(libc.to_string());
} else {
parts.push("any".to_string());
}
write!(f, "{}", parts.join("-"))
}
}
impl FromStr for PythonDownloadRequest {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut parts = s.split('-');
let mut version = None;
let mut implementation = None;
let mut os = None;
let mut arch = None;
let mut libc = None;
loop {
// Consume each part
let Some(part) = parts.next() else { break };
if implementation.is_none() {
implementation = Some(ImplementationName::from_str(part)?);
continue;
}
if version.is_none() {
version = Some(
VersionRequest::from_str(part)
.map_err(|_| Error::InvalidPythonVersion(part.to_string()))?,
);
continue;
}
if os.is_none() {
os = Some(Os::from_str(part)?);
continue;
}
if arch.is_none() {
arch = Some(Arch::from_str(part)?);
continue;
}
if libc.is_none() {
libc = Some(Libc::from_str(part)?);
continue;
}
return Err(Error::TooManyParts(s.to_string()));
}
Ok(Self::new(version, implementation, arch, os, libc))
}
}
include!("downloads.inc");
pub enum DownloadResult {
AlreadyAvailable(PathBuf),
Fetched(PathBuf),
}
impl ManagedPythonDownload {
/// Return the first [`PythonDownload`] matching a request, if any.
pub fn from_request(
request: &PythonDownloadRequest,
) -> Result<&'static ManagedPythonDownload, Error> {
request
.iter_downloads()
.next()
.ok_or(Error::NoDownloadFound(request.clone()))
}
/// Iterate over all [`PythonDownload`]'s.
pub fn iter_all() -> impl Iterator<Item = &'static ManagedPythonDownload> {
PYTHON_DOWNLOADS.iter()
}
pub fn url(&self) -> &str {
self.url
}
pub fn key(&self) -> &PythonInstallationKey {
&self.key
}
pub fn os(&self) -> &Os {
self.key.os()
}
pub fn sha256(&self) -> Option<&str> {
self.sha256
}
/// Download and extract
#[instrument(skip(client, parent_path), fields(download = %self.key()))]
pub async fn fetch(
&self,
client: &uv_client::BaseClient,
parent_path: &Path,
) -> Result<DownloadResult, Error> {
let url = Url::parse(self.url)?;
let path = parent_path.join(self.key().to_string()).clone();
// If it already exists, return it
if path.is_dir() {
return Ok(DownloadResult::AlreadyAvailable(path));
}
let filename = url.path_segments().unwrap().last().unwrap();
let response = client.get(url.clone()).send().await?;
// Ensure the request was successful.
response.error_for_status_ref()?;
// Download and extract into a temporary directory.
let temp_dir = tempfile::tempdir_in(parent_path).map_err(Error::DownloadDirError)?;
debug!(
"Downloading {url} to temporary location {}",
temp_dir.path().display()
);
let reader = response
.bytes_stream()
.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
.into_async_read();
debug!("Extracting {filename}");
uv_extract::stream::archive(reader.compat(), filename, temp_dir.path())
.await
.map_err(|err| Error::ExtractError(filename.to_string(), err))?;
// Extract the top-level directory.
let extracted = match uv_extract::strip_component(temp_dir.path()) {
Ok(top_level) => top_level,
Err(uv_extract::Error::NonSingularArchive(_)) => temp_dir.into_path(),
Err(err) => return Err(Error::ExtractError(filename.to_string(), err)),
};
// Persist it to the target
debug!("Moving {} to {}", extracted.display(), path.user_display());
rename_with_retry(extracted, &path)
.await
.map_err(|err| Error::CopyError {
to: path.clone(),
err,
})?;
Ok(DownloadResult::Fetched(path))
}
pub fn python_version(&self) -> PythonVersion {
self.key.version()
}
}
impl From<reqwest::Error> for Error {
fn from(error: reqwest::Error) -> Self {
Self::NetworkError(WrappedReqwestError::from(error))
}
}
impl From<reqwest_middleware::Error> for Error {
fn from(error: reqwest_middleware::Error) -> Self {
match error {
reqwest_middleware::Error::Middleware(error) => Self::NetworkMiddlewareError(error),
reqwest_middleware::Error::Reqwest(error) => {
Self::NetworkError(WrappedReqwestError::from(error))
}
}
}
}
impl Display for ManagedPythonDownload {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.key)
}
}

View file

@ -0,0 +1,233 @@
use std::borrow::Cow;
use std::env;
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use uv_cache::Cache;
use uv_fs::{LockedFile, Simplified};
use crate::discovery::find_python_installation;
use crate::installation::PythonInstallation;
use crate::virtualenv::{virtualenv_python_executable, PyVenvConfiguration};
use crate::{
EnvironmentPreference, Error, Interpreter, Prefix, PythonNotFound, PythonPreference,
PythonRequest, Target,
};
/// A Python environment, consisting of a Python [`Interpreter`] and its associated paths.
#[derive(Debug, Clone)]
pub struct PythonEnvironment(Arc<PythonEnvironmentShared>);
#[derive(Debug, Clone)]
struct PythonEnvironmentShared {
root: PathBuf,
interpreter: Interpreter,
}
/// The result of failed environment discovery.
///
/// Generally this is cast from [`PythonNotFound`] by [`PythonEnvironment::find`].
#[derive(Clone, Debug, Error)]
pub struct EnvironmentNotFound {
request: PythonRequest,
preference: EnvironmentPreference,
}
impl From<PythonNotFound> for EnvironmentNotFound {
fn from(value: PythonNotFound) -> Self {
Self {
request: value.request,
preference: value.environment_preference,
}
}
}
impl fmt::Display for EnvironmentNotFound {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let environment = match self.preference {
EnvironmentPreference::Any => "virtual or system environment",
EnvironmentPreference::ExplicitSystem => {
if self.request.is_explicit_system() {
"virtual or system environment"
} else {
// TODO(zanieb): We could add a hint to use the `--system` flag here
"virtual environment"
}
}
EnvironmentPreference::OnlySystem => "system environment",
EnvironmentPreference::OnlyVirtual => "virtual environment",
};
match self.request {
PythonRequest::Any => {
write!(f, "No {environment} found")
}
_ => {
write!(f, "No {environment} found for {}", self.request)
}
}
}
}
impl PythonEnvironment {
/// Find a [`PythonEnvironment`] matching the given request and preference.
///
/// If looking for a Python interpreter to create a new environment, use [`PythonInstallation::find`]
/// instead.
pub fn find(
request: &PythonRequest,
preference: EnvironmentPreference,
cache: &Cache,
) -> Result<Self, Error> {
let installation = match find_python_installation(
request,
preference,
// Ignore managed installations when looking for environments
PythonPreference::OnlySystem,
cache,
)? {
Ok(installation) => installation,
Err(err) => return Err(EnvironmentNotFound::from(err).into()),
};
Ok(Self::from_installation(installation))
}
/// Create a [`PythonEnvironment`] from the virtual environment at the given root.
pub fn from_root(root: impl AsRef<Path>, cache: &Cache) -> Result<Self, Error> {
let venv = match fs_err::canonicalize(root.as_ref()) {
Ok(venv) => venv,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::MissingEnvironment(EnvironmentNotFound {
preference: EnvironmentPreference::Any,
request: PythonRequest::Directory(root.as_ref().to_owned()),
}));
}
Err(err) => return Err(Error::Discovery(err.into())),
};
let executable = virtualenv_python_executable(venv);
let interpreter = Interpreter::query(executable, cache)?;
Ok(Self(Arc::new(PythonEnvironmentShared {
root: interpreter.sys_prefix().to_path_buf(),
interpreter,
})))
}
/// Create a [`PythonEnvironment`] from an existing [`PythonInstallation`].
pub fn from_installation(installation: PythonInstallation) -> Self {
Self::from_interpreter(installation.into_interpreter())
}
/// Create a [`PythonEnvironment`] from an existing [`Interpreter`].
pub fn from_interpreter(interpreter: Interpreter) -> Self {
Self(Arc::new(PythonEnvironmentShared {
root: interpreter.sys_prefix().to_path_buf(),
interpreter,
}))
}
/// Create a [`PythonEnvironment`] from an existing [`Interpreter`] and `--target` directory.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
let inner = Arc::unwrap_or_clone(self.0);
Self(Arc::new(PythonEnvironmentShared {
interpreter: inner.interpreter.with_target(target),
..inner
}))
}
/// Create a [`PythonEnvironment`] from an existing [`Interpreter`] and `--prefix` directory.
#[must_use]
pub fn with_prefix(self, prefix: Prefix) -> Self {
let inner = Arc::unwrap_or_clone(self.0);
Self(Arc::new(PythonEnvironmentShared {
interpreter: inner.interpreter.with_prefix(prefix),
..inner
}))
}
/// Returns the root (i.e., `prefix`) of the Python interpreter.
pub fn root(&self) -> &Path {
&self.0.root
}
/// Return the [`Interpreter`] for this virtual environment.
///
/// See also [`PythonEnvironment::into_interpreter`].
pub fn interpreter(&self) -> &Interpreter {
&self.0.interpreter
}
/// Return the [`PyVenvConfiguration`] for this environment, as extracted from the
/// `pyvenv.cfg` file.
pub fn cfg(&self) -> Result<PyVenvConfiguration, Error> {
Ok(PyVenvConfiguration::parse(self.0.root.join("pyvenv.cfg"))?)
}
/// Returns the location of the Python executable.
pub fn python_executable(&self) -> &Path {
self.0.interpreter.sys_executable()
}
/// Returns an iterator over the `site-packages` directories inside the environment.
///
/// In most cases, `purelib` and `platlib` will be the same, and so the iterator will contain
/// a single element; however, in some distributions, they may be different.
///
/// Some distributions also create symbolic links from `purelib` to `platlib`; in such cases, we
/// still deduplicate the entries, returning a single path.
pub fn site_packages(&self) -> impl Iterator<Item = Cow<Path>> {
let target = self.0.interpreter.target().map(Target::site_packages);
let prefix = self
.0
.interpreter
.prefix()
.map(|prefix| prefix.site_packages(self.0.interpreter.virtualenv()));
let interpreter = if target.is_none() && prefix.is_none() {
Some(self.0.interpreter.site_packages())
} else {
None
};
target
.into_iter()
.flatten()
.map(Cow::Borrowed)
.chain(prefix.into_iter().flatten().map(Cow::Owned))
.chain(interpreter.into_iter().flatten().map(Cow::Borrowed))
}
/// Returns the path to the `bin` directory inside this environment.
pub fn scripts(&self) -> &Path {
self.0.interpreter.scripts()
}
/// Grab a file lock for the environment to prevent concurrent writes across processes.
pub fn lock(&self) -> Result<LockedFile, std::io::Error> {
if let Some(target) = self.0.interpreter.target() {
// If we're installing into a `--target`, use a target-specific lock file.
LockedFile::acquire(target.root().join(".lock"), target.root().user_display())
} else if let Some(prefix) = self.0.interpreter.prefix() {
// Likewise, if we're installing into a `--prefix`, use a prefix-specific lock file.
LockedFile::acquire(prefix.root().join(".lock"), prefix.root().user_display())
} else if self.0.interpreter.is_virtualenv() {
// If the environment a virtualenv, use a virtualenv-specific lock file.
LockedFile::acquire(self.0.root.join(".lock"), self.0.root.user_display())
} else {
// Otherwise, use a global lock file.
LockedFile::acquire(
env::temp_dir().join(format!("uv-{}.lock", cache_key::digest(&self.0.root))),
self.0.root.user_display(),
)
}
}
/// Return the [`Interpreter`] for this environment.
///
/// See also [`PythonEnvironment::interpreter`].
pub fn into_interpreter(self) -> Interpreter {
Arc::unwrap_or_clone(self.0).interpreter
}
}

View file

@ -0,0 +1,109 @@
use std::{
fmt::{self, Display},
str::FromStr,
};
use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("Unknown Python implementation `{0}`")]
UnknownImplementation(String),
}
#[derive(Debug, Eq, PartialEq, Clone, Copy, Default, PartialOrd, Ord)]
pub enum ImplementationName {
#[default]
CPython,
PyPy,
}
#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd)]
pub enum LenientImplementationName {
Known(ImplementationName),
Unknown(String),
}
impl ImplementationName {
pub(crate) fn possible_names() -> impl Iterator<Item = &'static str> {
["cpython", "pypy", "cp", "pp"].into_iter()
}
pub fn pretty(self) -> &'static str {
match self {
Self::CPython => "CPython",
Self::PyPy => "PyPy",
}
}
}
impl LenientImplementationName {
pub fn pretty(&self) -> &str {
match self {
Self::Known(implementation) => implementation.pretty(),
Self::Unknown(name) => name,
}
}
}
impl From<&ImplementationName> for &'static str {
fn from(v: &ImplementationName) -> &'static str {
match v {
ImplementationName::CPython => "cpython",
ImplementationName::PyPy => "pypy",
}
}
}
impl<'a> From<&'a LenientImplementationName> for &'a str {
fn from(v: &'a LenientImplementationName) -> &'a str {
match v {
LenientImplementationName::Known(implementation) => implementation.into(),
LenientImplementationName::Unknown(name) => name,
}
}
}
impl FromStr for ImplementationName {
type Err = Error;
/// Parse a Python implementation name from a string.
///
/// Supports the full name and the platform compatibility tag style name.
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_ascii_lowercase().as_str() {
"cpython" | "cp" => Ok(Self::CPython),
"pypy" | "pp" => Ok(Self::PyPy),
_ => Err(Error::UnknownImplementation(s.to_string())),
}
}
}
impl Display for ImplementationName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.into())
}
}
impl From<&str> for LenientImplementationName {
fn from(s: &str) -> Self {
match ImplementationName::from_str(s) {
Ok(implementation) => Self::Known(implementation),
Err(_) => Self::Unknown(s.to_string()),
}
}
}
impl From<ImplementationName> for LenientImplementationName {
fn from(implementation: ImplementationName) -> Self {
Self::Known(implementation)
}
}
impl Display for LenientImplementationName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Known(implementation) => implementation.fmt(f),
Self::Unknown(name) => f.write_str(&name.to_ascii_lowercase()),
}
}
}

View file

@ -0,0 +1,342 @@
use std::fmt;
use std::str::FromStr;
use pep440_rs::Version;
use tracing::{debug, info};
use uv_client::BaseClientBuilder;
use uv_cache::Cache;
use crate::discovery::{
find_best_python_installation, find_python_installation, EnvironmentPreference, PythonRequest,
};
use crate::downloads::{DownloadResult, ManagedPythonDownload, PythonDownloadRequest};
use crate::implementation::LenientImplementationName;
use crate::managed::{ManagedPythonInstallation, ManagedPythonInstallations};
use crate::platform::{Arch, Libc, Os};
use crate::{Error, Interpreter, PythonFetch, PythonPreference, PythonSource, PythonVersion};
/// A Python interpreter and accompanying tools.
#[derive(Clone, Debug)]
pub struct PythonInstallation {
// Public in the crate for test assertions
pub(crate) source: PythonSource,
pub(crate) interpreter: Interpreter,
}
impl PythonInstallation {
/// Create a new [`PythonInstallation`] from a source, interpreter tuple.
pub(crate) fn from_tuple(tuple: (PythonSource, Interpreter)) -> Self {
let (source, interpreter) = tuple;
Self {
source,
interpreter,
}
}
/// Find an installed [`PythonInstallation`].
///
/// This is the standard interface for discovering a Python installation for creating
/// an environment. If interested in finding an existing environment, see
/// [`PythonEnvironment::find`] instead.
///
/// Note we still require an [`EnvironmentPreference`] as this can either bypass virtual environments
/// or prefer them. In most cases, this should be [`EnvironmentPreference::OnlySystem`]
/// but if you want to allow an interpreter from a virtual environment if it satisfies the request,
/// then use [`EnvironmentPreference::Any`].
///
/// See [`find_installation`] for implementation details.
pub fn find(
request: &PythonRequest,
environments: EnvironmentPreference,
preference: PythonPreference,
cache: &Cache,
) -> Result<Self, Error> {
let installation = find_python_installation(request, environments, preference, cache)??;
Ok(installation)
}
/// Find an installed [`PythonInstallation`] that satisfies a requested version, if the request cannot
/// be satisfied, fallback to the best available Python installation.
pub fn find_best(
request: &PythonRequest,
environments: EnvironmentPreference,
preference: PythonPreference,
cache: &Cache,
) -> Result<Self, Error> {
Ok(find_best_python_installation(
request,
environments,
preference,
cache,
)??)
}
/// Find or fetch a [`PythonInstallation`].
///
/// Unlike [`PythonInstallation::find`], if the required Python is not installed it will be installed automatically.
pub async fn find_or_fetch<'a>(
request: Option<PythonRequest>,
environments: EnvironmentPreference,
preference: PythonPreference,
python_fetch: PythonFetch,
client_builder: &BaseClientBuilder<'a>,
cache: &Cache,
) -> Result<Self, Error> {
let request = request.unwrap_or_default();
// Perform a fetch aggressively if managed Python is preferred
if matches!(preference, PythonPreference::Managed) && python_fetch.is_automatic() {
if let Some(request) = PythonDownloadRequest::try_from_request(&request) {
return Self::fetch(request, client_builder, cache).await;
}
}
// Search for the installation
match Self::find(&request, environments, preference, cache) {
Ok(venv) => Ok(venv),
// If missing and allowed, perform a fetch
err @ Err(Error::MissingPython(_))
if preference.allows_managed()
&& python_fetch.is_automatic()
&& client_builder.connectivity.is_online() =>
{
if let Some(request) = PythonDownloadRequest::try_from_request(&request) {
debug!("Requested Python not found, checking for available download...");
Self::fetch(request, client_builder, cache).await
} else {
err
}
}
Err(err) => Err(err),
}
}
/// Download and install the requested installation.
pub async fn fetch<'a>(
request: PythonDownloadRequest,
client_builder: &BaseClientBuilder<'a>,
cache: &Cache,
) -> Result<Self, Error> {
let installations = ManagedPythonInstallations::from_settings()?.init()?;
let installations_dir = installations.root();
let _lock = installations.acquire_lock()?;
let download = ManagedPythonDownload::from_request(&request)?;
let client = client_builder.build();
info!("Fetching requested Python...");
let result = download.fetch(&client, installations_dir).await?;
let path = match result {
DownloadResult::AlreadyAvailable(path) => path,
DownloadResult::Fetched(path) => path,
};
let installed = ManagedPythonInstallation::new(path)?;
installed.ensure_externally_managed()?;
Ok(Self {
source: PythonSource::Managed,
interpreter: Interpreter::query(installed.executable(), cache)?,
})
}
/// Create a [`PythonInstallation`] from an existing [`Interpreter`].
pub fn from_interpreter(interpreter: Interpreter) -> Self {
Self {
source: PythonSource::ProvidedPath,
interpreter,
}
}
/// Return the [`PythonSource`] of the Python installation, indicating where it was found.
pub fn source(&self) -> &PythonSource {
&self.source
}
pub fn key(&self) -> PythonInstallationKey {
PythonInstallationKey::new(
LenientImplementationName::from(self.interpreter.implementation_name()),
self.interpreter.python_major(),
self.interpreter.python_minor(),
self.interpreter.python_patch(),
self.os(),
self.arch(),
self.libc(),
)
}
/// Return the Python [`Version`] of the Python installation as reported by its interpreter.
pub fn python_version(&self) -> &Version {
self.interpreter.python_version()
}
/// Return the [`LenientImplementationName`] of the Python installation as reported by its interpreter.
pub fn implementation(&self) -> LenientImplementationName {
LenientImplementationName::from(self.interpreter.implementation_name())
}
/// Return the [`Arch`] of the Python installation as reported by its interpreter.
pub fn arch(&self) -> Arch {
Arch::from(&self.interpreter.platform().arch())
}
/// Return the [`Libc`] of the Python installation as reported by its interpreter.
pub fn libc(&self) -> Libc {
Libc::from(self.interpreter.platform().os())
}
/// Return the [`Os`] of the Python installation as reported by its interpreter.
pub fn os(&self) -> Os {
Os::from(self.interpreter.platform().os())
}
/// Return the [`Interpreter`] for the Python installation.
pub fn interpreter(&self) -> &Interpreter {
&self.interpreter
}
pub fn into_interpreter(self) -> Interpreter {
self.interpreter
}
}
#[derive(Error, Debug)]
pub enum PythonInstallationKeyError {
#[error("Failed to parse Python installation key `{0}`: {1}")]
ParseError(String, String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PythonInstallationKey {
pub(crate) implementation: LenientImplementationName,
pub(crate) major: u8,
pub(crate) minor: u8,
pub(crate) patch: u8,
pub(crate) os: Os,
pub(crate) arch: Arch,
pub(crate) libc: Libc,
}
impl PythonInstallationKey {
pub fn new(
implementation: LenientImplementationName,
major: u8,
minor: u8,
patch: u8,
os: Os,
arch: Arch,
libc: Libc,
) -> Self {
Self {
implementation,
major,
minor,
patch,
os,
arch,
libc,
}
}
pub fn implementation(&self) -> &LenientImplementationName {
&self.implementation
}
pub fn version(&self) -> PythonVersion {
PythonVersion::from_str(&format!("{}.{}.{}", self.major, self.minor, self.patch))
.expect("Python installation keys must have valid Python versions")
}
pub fn arch(&self) -> &Arch {
&self.arch
}
pub fn os(&self) -> &Os {
&self.os
}
pub fn libc(&self) -> &Libc {
&self.libc
}
}
impl fmt::Display for PythonInstallationKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{}-{}.{}.{}-{}-{}-{}",
self.implementation, self.major, self.minor, self.patch, self.os, self.arch, self.libc
)
}
}
impl FromStr for PythonInstallationKey {
type Err = PythonInstallationKeyError;
fn from_str(key: &str) -> Result<Self, Self::Err> {
let parts = key.split('-').collect::<Vec<_>>();
let [implementation, version, os, arch, libc] = parts.as_slice() else {
return Err(PythonInstallationKeyError::ParseError(
key.to_string(),
"not enough `-`-separated values".to_string(),
));
};
let implementation = LenientImplementationName::from(*implementation);
let os = Os::from_str(os).map_err(|err| {
PythonInstallationKeyError::ParseError(key.to_string(), format!("invalid OS: {err}"))
})?;
let arch = Arch::from_str(arch).map_err(|err| {
PythonInstallationKeyError::ParseError(
key.to_string(),
format!("invalid architecture: {err}"),
)
})?;
let libc = Libc::from_str(libc).map_err(|err| {
PythonInstallationKeyError::ParseError(key.to_string(), format!("invalid libc: {err}"))
})?;
let [major, minor, patch] = version
.splitn(3, '.')
.map(str::parse::<u8>)
.collect::<Result<Vec<_>, _>>()
.map_err(|err| {
PythonInstallationKeyError::ParseError(
key.to_string(),
format!("invalid Python version: {err}"),
)
})?[..]
else {
return Err(PythonInstallationKeyError::ParseError(
key.to_string(),
"invalid Python version: expected `<major>.<minor>.<patch>`".to_string(),
));
};
Ok(Self::new(
implementation,
major,
minor,
patch,
os,
arch,
libc,
))
}
}
impl PartialOrd for PythonInstallationKey {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for PythonInstallationKey {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.to_string().cmp(&other.to_string())
}
}

View file

@ -0,0 +1,820 @@
use std::io;
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus};
use configparser::ini::Ini;
use fs_err as fs;
use once_cell::sync::OnceCell;
use same_file::is_same_file;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::{trace, warn};
use cache_key::digest;
use install_wheel_rs::Layout;
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, StringVersion};
use platform_tags::Platform;
use platform_tags::{Tags, TagsError};
use pypi_types::Scheme;
use uv_cache::{Cache, CacheBucket, CachedByTimestamp, Freshness, Timestamp};
use uv_fs::{write_atomic_sync, PythonExt, Simplified};
use crate::pointer_size::PointerSize;
use crate::{Prefix, PythonVersion, Target, VirtualEnvironment};
/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
pub struct Interpreter {
platform: Platform,
markers: Box<MarkerEnvironment>,
scheme: Scheme,
virtualenv: Scheme,
sys_prefix: PathBuf,
sys_base_exec_prefix: PathBuf,
sys_base_prefix: PathBuf,
sys_base_executable: Option<PathBuf>,
sys_executable: PathBuf,
sys_path: Vec<PathBuf>,
stdlib: PathBuf,
tags: OnceCell<Tags>,
target: Option<Target>,
prefix: Option<Prefix>,
pointer_size: PointerSize,
gil_disabled: bool,
}
impl Interpreter {
/// Detect the interpreter info for the given Python executable.
pub fn query(executable: impl AsRef<Path>, cache: &Cache) -> Result<Self, Error> {
let info = InterpreterInfo::query_cached(executable.as_ref(), cache)?;
debug_assert!(
info.sys_executable.is_absolute(),
"`sys.executable` is not an absolute Python; Python installation is broken: {}",
info.sys_executable.display()
);
Ok(Self {
platform: info.platform,
markers: Box::new(info.markers),
scheme: info.scheme,
virtualenv: info.virtualenv,
sys_prefix: info.sys_prefix,
sys_base_exec_prefix: info.sys_base_exec_prefix,
pointer_size: info.pointer_size,
gil_disabled: info.gil_disabled,
sys_base_prefix: info.sys_base_prefix,
sys_base_executable: info.sys_base_executable,
sys_executable: info.sys_executable,
sys_path: info.sys_path,
stdlib: info.stdlib,
tags: OnceCell::new(),
target: None,
prefix: None,
})
}
// TODO(konstin): Find a better way mocking the fields
pub fn artificial(platform: Platform, markers: MarkerEnvironment) -> Self {
Self {
platform,
markers: Box::new(markers),
scheme: Scheme {
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
scripts: PathBuf::from("/dev/null"),
data: PathBuf::from("/dev/null"),
},
virtualenv: Scheme {
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
scripts: PathBuf::from("/dev/null"),
data: PathBuf::from("/dev/null"),
},
sys_prefix: PathBuf::from("/dev/null"),
sys_base_exec_prefix: PathBuf::from("/dev/null"),
sys_base_prefix: PathBuf::from("/dev/null"),
sys_base_executable: None,
sys_executable: PathBuf::from("/dev/null"),
sys_path: vec![],
stdlib: PathBuf::from("/dev/null"),
tags: OnceCell::new(),
target: None,
prefix: None,
pointer_size: PointerSize::_64,
gil_disabled: false,
}
}
/// Return a new [`Interpreter`] with the given virtual environment root.
#[must_use]
pub fn with_virtualenv(self, virtualenv: VirtualEnvironment) -> Self {
Self {
scheme: virtualenv.scheme,
sys_executable: virtualenv.executable,
sys_prefix: virtualenv.root,
target: None,
prefix: None,
..self
}
}
/// Return a new [`Interpreter`] to install into the given `--target` directory.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
Self {
target: Some(target),
..self
}
}
/// Return a new [`Interpreter`] to install into the given `--prefix` directory.
#[must_use]
pub fn with_prefix(self, prefix: Prefix) -> Self {
Self {
prefix: Some(prefix),
..self
}
}
/// Returns the path to the Python virtual environment.
#[inline]
pub fn platform(&self) -> &Platform {
&self.platform
}
/// Returns the [`MarkerEnvironment`] for this Python executable.
#[inline]
pub const fn markers(&self) -> &MarkerEnvironment {
&self.markers
}
/// Returns the [`Tags`] for this Python executable.
pub fn tags(&self) -> Result<&Tags, TagsError> {
self.tags.get_or_try_init(|| {
Tags::from_env(
self.platform(),
self.python_tuple(),
self.implementation_name(),
self.implementation_tuple(),
self.gil_disabled,
)
})
}
/// Returns `true` if the environment is a PEP 405-compliant virtual environment.
///
/// See: <https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/utils/virtualenv.py#L14>
pub fn is_virtualenv(&self) -> bool {
// Maybe this should return `false` if it's a target?
self.sys_prefix != self.sys_base_prefix
}
/// Returns `true` if the environment is a `--target` environment.
pub fn is_target(&self) -> bool {
self.target.is_some()
}
/// Returns `true` if the environment is a `--prefix` environment.
pub fn is_prefix(&self) -> bool {
self.prefix.is_some()
}
/// Returns `Some` if the environment is externally managed, optionally including an error
/// message from the `EXTERNALLY-MANAGED` file.
///
/// See: <https://packaging.python.org/en/latest/specifications/externally-managed-environments/>
pub fn is_externally_managed(&self) -> Option<ExternallyManaged> {
// Per the spec, a virtual environment is never externally managed.
if self.is_virtualenv() {
return None;
}
// If we're installing into a target or prefix directory, it's never externally managed.
if self.is_target() || self.is_prefix() {
return None;
}
let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
return None;
};
let mut ini = Ini::new_cs();
ini.set_multiline(true);
let Ok(mut sections) = ini.read(contents) else {
// If a file exists but is not a valid INI file, we assume the environment is
// externally managed.
return Some(ExternallyManaged::default());
};
let Some(section) = sections.get_mut("externally-managed") else {
// If the file exists but does not contain an "externally-managed" section, we assume
// the environment is externally managed.
return Some(ExternallyManaged::default());
};
let Some(error) = section.remove("Error") else {
// If the file exists but does not contain an "Error" key, we assume the environment is
// externally managed.
return Some(ExternallyManaged::default());
};
Some(ExternallyManaged { error })
}
/// Returns the `python_full_version` marker corresponding to this Python version.
#[inline]
pub fn python_full_version(&self) -> &StringVersion {
self.markers.python_full_version()
}
/// Returns the full Python version.
#[inline]
pub fn python_version(&self) -> &Version {
&self.markers.python_full_version().version
}
/// Returns the full minor Python version.
#[inline]
pub fn python_minor_version(&self) -> Version {
Version::new(self.python_version().release().iter().take(2).copied())
}
/// Return the major version of this Python version.
pub fn python_major(&self) -> u8 {
let major = self.markers.python_full_version().version.release()[0];
u8::try_from(major).expect("invalid major version")
}
/// Return the minor version of this Python version.
pub fn python_minor(&self) -> u8 {
let minor = self.markers.python_full_version().version.release()[1];
u8::try_from(minor).expect("invalid minor version")
}
/// Return the patch version of this Python version.
pub fn python_patch(&self) -> u8 {
let minor = self.markers.python_full_version().version.release()[2];
u8::try_from(minor).expect("invalid patch version")
}
/// Returns the Python version as a simple tuple.
pub fn python_tuple(&self) -> (u8, u8) {
(self.python_major(), self.python_minor())
}
/// Return the major version of the implementation (e.g., `CPython` or `PyPy`).
pub fn implementation_major(&self) -> u8 {
let major = self.markers.implementation_version().version.release()[0];
u8::try_from(major).expect("invalid major version")
}
/// Return the minor version of the implementation (e.g., `CPython` or `PyPy`).
pub fn implementation_minor(&self) -> u8 {
let minor = self.markers.implementation_version().version.release()[1];
u8::try_from(minor).expect("invalid minor version")
}
/// Returns the implementation version as a simple tuple.
pub fn implementation_tuple(&self) -> (u8, u8) {
(self.implementation_major(), self.implementation_minor())
}
/// Returns the implementation name (e.g., `CPython` or `PyPy`).
pub fn implementation_name(&self) -> &str {
self.markers.implementation_name()
}
/// Return the `sys.base_exec_prefix` path for this Python interpreter.
pub fn sys_base_exec_prefix(&self) -> &Path {
&self.sys_base_exec_prefix
}
/// Return the `sys.base_prefix` path for this Python interpreter.
pub fn sys_base_prefix(&self) -> &Path {
&self.sys_base_prefix
}
/// Return the `sys.prefix` path for this Python interpreter.
pub fn sys_prefix(&self) -> &Path {
&self.sys_prefix
}
/// Return the `sys._base_executable` path for this Python interpreter. Some platforms do not
/// have this attribute, so it may be `None`.
pub fn sys_base_executable(&self) -> Option<&Path> {
self.sys_base_executable.as_deref()
}
/// Return the `sys.executable` path for this Python interpreter.
pub fn sys_executable(&self) -> &Path {
&self.sys_executable
}
/// Return the `sys.path` for this Python interpreter.
pub fn sys_path(&self) -> &Vec<PathBuf> {
&self.sys_path
}
/// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn stdlib(&self) -> &Path {
&self.stdlib
}
/// Return the `purelib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn purelib(&self) -> &Path {
&self.scheme.purelib
}
/// Return the `platlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn platlib(&self) -> &Path {
&self.scheme.platlib
}
/// Return the `scripts` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn scripts(&self) -> &Path {
&self.scheme.scripts
}
/// Return the `data` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn data(&self) -> &Path {
&self.scheme.data
}
/// Return the `include` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn include(&self) -> &Path {
&self.scheme.include
}
/// Return the [`Scheme`] for a virtual environment created by this [`Interpreter`].
pub fn virtualenv(&self) -> &Scheme {
&self.virtualenv
}
/// Return the [`PointerSize`] of the Python interpreter (i.e., 32- vs. 64-bit).
pub fn pointer_size(&self) -> PointerSize {
self.pointer_size
}
/// Return whether this is a Python 3.13+ freethreading Python, as specified by the sysconfig var
/// `Py_GIL_DISABLED`.
///
/// freethreading Python is incompatible with earlier native modules, re-introducing
/// abiflags with a `t` flag. <https://peps.python.org/pep-0703/#build-configuration-changes>
pub fn gil_disabled(&self) -> bool {
self.gil_disabled
}
/// Return the `--target` directory for this interpreter, if any.
pub fn target(&self) -> Option<&Target> {
self.target.as_ref()
}
/// Return the `--prefix` directory for this interpreter, if any.
pub fn prefix(&self) -> Option<&Prefix> {
self.prefix.as_ref()
}
/// Return the [`Layout`] environment used to install wheels into this interpreter.
pub fn layout(&self) -> Layout {
Layout {
python_version: self.python_tuple(),
sys_executable: self.sys_executable().to_path_buf(),
os_name: self.markers.os_name().to_string(),
scheme: if let Some(target) = self.target.as_ref() {
target.scheme()
} else if let Some(prefix) = self.prefix.as_ref() {
prefix.scheme(&self.virtualenv)
} else {
Scheme {
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
data: self.data().to_path_buf(),
include: if self.is_virtualenv() {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.sys_prefix.join("include").join("site").join(format!(
"python{}.{}",
self.python_major(),
self.python_minor()
))
} else {
self.include().to_path_buf()
},
}
},
}
}
/// Return an iterator over the `site-packages` directories inside the environment.
pub fn site_packages(&self) -> impl Iterator<Item = &Path> {
let purelib = self.purelib();
let platlib = self.platlib();
std::iter::once(purelib).chain(
if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) {
None
} else {
Some(platlib)
},
)
}
/// Check if the interpreter matches the given Python version.
///
/// If a patch version is present, we will require an exact match.
/// Otherwise, just the major and minor version numbers need to match.
pub fn satisfies(&self, version: &PythonVersion) -> bool {
if version.patch().is_some() {
version.version() == self.python_version()
} else {
(version.major(), version.minor()) == self.python_tuple()
}
}
}
/// The `EXTERNALLY-MANAGED` file in a Python installation.
///
/// See: <https://packaging.python.org/en/latest/specifications/externally-managed-environments/>
#[derive(Debug, Default, Clone)]
pub struct ExternallyManaged {
error: Option<String>,
}
impl ExternallyManaged {
/// Return the `EXTERNALLY-MANAGED` error message, if any.
pub fn into_error(self) -> Option<String> {
self.error
}
}
#[derive(Debug, Error)]
pub enum Error {
#[error("Failed to query Python interpreter")]
Io(#[from] io::Error),
#[error("Python interpreter not found at `{0}`")]
NotFound(PathBuf),
#[error("Failed to query Python interpreter at `{path}`")]
SpawnFailed {
path: PathBuf,
#[source]
err: io::Error,
},
#[error("Querying Python at `{}` did not return the expected data\n{err}\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---", path.display())]
UnexpectedResponse {
err: serde_json::Error,
stdout: String,
stderr: String,
path: PathBuf,
},
#[error("Querying Python at `{}` failed with exit status {code}\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---", path.display())]
StatusCode {
code: ExitStatus,
stdout: String,
stderr: String,
path: PathBuf,
},
#[error("Can't use Python at `{path}`")]
QueryScript {
#[source]
err: InterpreterInfoError,
path: PathBuf,
},
#[error("Failed to write to cache")]
Encode(#[from] rmp_serde::encode::Error),
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "result", rename_all = "lowercase")]
enum InterpreterInfoResult {
Error(InterpreterInfoError),
Success(Box<InterpreterInfo>),
}
#[derive(Debug, Error, Deserialize, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum InterpreterInfoError {
#[error("Could not detect a glibc or a musl libc (while running on Linux)")]
LibcNotFound,
#[error("Unknown operation system: `{operating_system}`")]
UnknownOperatingSystem { operating_system: String },
#[error("Python {python_version} is not supported. Please use Python 3.8 or newer.")]
UnsupportedPythonVersion { python_version: String },
#[error("Python executable does not support `-I` flag. Please use Python 3.8 or newer.")]
UnsupportedPython,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
struct InterpreterInfo {
platform: Platform,
markers: MarkerEnvironment,
scheme: Scheme,
virtualenv: Scheme,
sys_prefix: PathBuf,
sys_base_exec_prefix: PathBuf,
sys_base_prefix: PathBuf,
sys_base_executable: Option<PathBuf>,
sys_executable: PathBuf,
sys_path: Vec<PathBuf>,
stdlib: PathBuf,
pointer_size: PointerSize,
gil_disabled: bool,
}
impl InterpreterInfo {
/// Return the resolved [`InterpreterInfo`] for the given Python executable.
pub(crate) fn query(interpreter: &Path, cache: &Cache) -> Result<Self, Error> {
let tempdir = tempfile::tempdir_in(cache.root())?;
Self::setup_python_query_files(tempdir.path())?;
// Sanitize the path by (1) running under isolated mode (`-I`) to ignore any site packages
// modifications, and then (2) adding the path containing our query script to the front of
// `sys.path` so that we can import it.
let script = format!(
r#"import sys; sys.path = ["{}"] + sys.path; from python.get_interpreter_info import main; main()"#,
tempdir.path().escape_for_python()
);
let output = Command::new(interpreter)
.arg("-I")
.arg("-c")
.arg(script)
.output()
.map_err(|err| Error::SpawnFailed {
path: interpreter.to_path_buf(),
err,
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
// If the Python version is too old, we may not even be able to invoke the query script
if stderr.contains("Unknown option: -I") {
return Err(Error::QueryScript {
err: InterpreterInfoError::UnsupportedPython,
path: interpreter.to_path_buf(),
});
}
return Err(Error::StatusCode {
code: output.status,
stderr,
stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
path: interpreter.to_path_buf(),
});
}
let result: InterpreterInfoResult =
serde_json::from_slice(&output.stdout).map_err(|err| {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
// If the Python version is too old, we may not even be able to invoke the query script
if stderr.contains("Unknown option: -I") {
Error::QueryScript {
err: InterpreterInfoError::UnsupportedPython,
path: interpreter.to_path_buf(),
}
} else {
Error::UnexpectedResponse {
err,
stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
stderr,
path: interpreter.to_path_buf(),
}
}
})?;
match result {
InterpreterInfoResult::Error(err) => Err(Error::QueryScript {
err,
path: interpreter.to_path_buf(),
}),
InterpreterInfoResult::Success(data) => Ok(*data),
}
}
/// Duplicate the directory structure we have in `../python` into a tempdir, so we can run
/// the Python probing scripts with `python -m python.get_interpreter_info` from that tempdir.
fn setup_python_query_files(root: &Path) -> Result<(), Error> {
let python_dir = root.join("python");
fs_err::create_dir(&python_dir)?;
fs_err::write(
python_dir.join("get_interpreter_info.py"),
include_str!("../python/get_interpreter_info.py"),
)?;
fs_err::write(
python_dir.join("__init__.py"),
include_str!("../python/__init__.py"),
)?;
let packaging_dir = python_dir.join("packaging");
fs_err::create_dir(&packaging_dir)?;
fs_err::write(
packaging_dir.join("__init__.py"),
include_str!("../python/packaging/__init__.py"),
)?;
fs_err::write(
packaging_dir.join("_elffile.py"),
include_str!("../python/packaging/_elffile.py"),
)?;
fs_err::write(
packaging_dir.join("_manylinux.py"),
include_str!("../python/packaging/_manylinux.py"),
)?;
fs_err::write(
packaging_dir.join("_musllinux.py"),
include_str!("../python/packaging/_musllinux.py"),
)?;
Ok(())
}
/// A wrapper around [`markers::query_interpreter_info`] to cache the computed markers.
///
/// Running a Python script is (relatively) expensive, and the markers won't change
/// unless the Python executable changes, so we use the executable's last modified
/// time as a cache key.
pub(crate) fn query_cached(executable: &Path, cache: &Cache) -> Result<Self, Error> {
let cache_entry = cache.entry(
CacheBucket::Interpreter,
"",
// We use the absolute path for the cache entry to avoid cache collisions for relative paths
// but we do not want to query the executable with symbolic links resolved
format!("{}.msgpack", digest(&uv_fs::absolutize_path(executable)?)),
);
// We check the timestamp of the canonicalized executable to check if an underlying
// interpreter has been modified
let modified =
Timestamp::from_path(uv_fs::canonicalize_executable(executable).map_err(|err| {
if err.kind() == io::ErrorKind::NotFound {
Error::NotFound(executable.to_path_buf())
} else {
err.into()
}
})?)?;
// Read from the cache.
if cache
.freshness(&cache_entry, None)
.is_ok_and(Freshness::is_fresh)
{
if let Ok(data) = fs::read(cache_entry.path()) {
match rmp_serde::from_slice::<CachedByTimestamp<Self>>(&data) {
Ok(cached) => {
if cached.timestamp == modified {
trace!(
"Cached interpreter info for Python {}, skipping probing: {}",
cached.data.markers.python_full_version(),
executable.user_display()
);
return Ok(cached.data);
}
trace!(
"Ignoring stale interpreter markers for: {}",
executable.user_display()
);
}
Err(err) => {
warn!(
"Broken interpreter cache entry at {}, removing: {err}",
cache_entry.path().user_display()
);
let _ = fs_err::remove_file(cache_entry.path());
}
}
}
}
// Otherwise, run the Python script.
trace!(
"Querying interpreter executable at {}",
executable.display()
);
let info = Self::query(executable, cache)?;
// If `executable` is a pyenv shim, a bash script that redirects to the activated
// python executable at another path, we're not allowed to cache the interpreter info.
if same_file::is_same_file(executable, &info.sys_executable).unwrap_or(false) {
fs::create_dir_all(cache_entry.dir())?;
write_atomic_sync(
cache_entry.path(),
rmp_serde::to_vec(&CachedByTimestamp {
timestamp: modified,
data: info.clone(),
})?,
)?;
}
Ok(info)
}
}
#[cfg(unix)]
#[cfg(test)]
mod tests {
use std::str::FromStr;
use fs_err as fs;
use indoc::{formatdoc, indoc};
use tempfile::tempdir;
use pep440_rs::Version;
use uv_cache::Cache;
use crate::Interpreter;
#[test]
fn test_cache_invalidation() {
let mock_dir = tempdir().unwrap();
let mocked_interpreter = mock_dir.path().join("python");
let json = indoc! {r##"
{
"result": "success",
"platform": {
"os": {
"name": "manylinux",
"major": 2,
"minor": 38
},
"arch": "x86_64"
},
"markers": {
"implementation_name": "cpython",
"implementation_version": "3.12.0",
"os_name": "posix",
"platform_machine": "x86_64",
"platform_python_implementation": "CPython",
"platform_release": "6.5.0-13-generic",
"platform_system": "Linux",
"platform_version": "#13-Ubuntu SMP PREEMPT_DYNAMIC Fri Nov 3 12:16:05 UTC 2023",
"python_full_version": "3.12.0",
"python_version": "3.12",
"sys_platform": "linux"
},
"sys_base_exec_prefix": "/home/ferris/.pyenv/versions/3.12.0",
"sys_base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
"sys_prefix": "/home/ferris/projects/uv/.venv",
"sys_executable": "/home/ferris/projects/uv/.venv/bin/python",
"sys_path": [
"/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/lib/python3.12",
"/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages"
],
"stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12",
"scheme": {
"data": "/home/ferris/.pyenv/versions/3.12.0",
"include": "/home/ferris/.pyenv/versions/3.12.0/include",
"platlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"purelib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"scripts": "/home/ferris/.pyenv/versions/3.12.0/bin"
},
"virtualenv": {
"data": "",
"include": "include",
"platlib": "lib/python3.12/site-packages",
"purelib": "lib/python3.12/site-packages",
"scripts": "bin"
},
"pointer_size": "64",
"gil_disabled": true
}
"##};
let cache = Cache::temp().unwrap().init().unwrap();
fs::write(
&mocked_interpreter,
formatdoc! {r##"
#!/bin/bash
echo '{json}'
"##},
)
.unwrap();
fs::set_permissions(
&mocked_interpreter,
std::os::unix::fs::PermissionsExt::from_mode(0o770),
)
.unwrap();
let interpreter = Interpreter::query(&mocked_interpreter, &cache).unwrap();
assert_eq!(
interpreter.markers.python_version().version,
Version::from_str("3.12").unwrap()
);
fs::write(
&mocked_interpreter,
formatdoc! {r##"
#!/bin/bash
echo '{}'
"##, json.replace("3.12", "3.13")},
)
.unwrap();
let interpreter = Interpreter::query(&mocked_interpreter, &cache).unwrap();
assert_eq!(
interpreter.markers.python_version().version,
Version::from_str("3.13").unwrap()
);
}
}

1870
crates/uv-python/src/lib.rs Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,342 @@
use core::fmt;
use fs_err as fs;
use std::collections::BTreeSet;
use std::ffi::OsStr;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use thiserror::Error;
use tracing::warn;
use uv_state::{StateBucket, StateStore};
use crate::downloads::Error as DownloadError;
use crate::implementation::{
Error as ImplementationError, ImplementationName, LenientImplementationName,
};
use crate::installation::{self, PythonInstallationKey};
use crate::platform::Error as PlatformError;
use crate::platform::{Arch, Libc, Os};
use crate::python_version::PythonVersion;
use crate::PythonRequest;
use uv_fs::{LockedFile, Simplified};
#[derive(Error, Debug)]
pub enum Error {
#[error(transparent)]
IO(#[from] io::Error),
#[error(transparent)]
Download(#[from] DownloadError),
#[error(transparent)]
PlatformError(#[from] PlatformError),
#[error(transparent)]
ImplementationError(#[from] ImplementationError),
#[error("Invalid python version: {0}")]
InvalidPythonVersion(String),
#[error(transparent)]
ExtractError(#[from] uv_extract::Error),
#[error("Failed to copy to: {0}", to.user_display())]
CopyError {
to: PathBuf,
#[source]
err: io::Error,
},
#[error("Failed to read Python installation directory: {0}", dir.user_display())]
ReadError {
dir: PathBuf,
#[source]
err: io::Error,
},
#[error("Failed to read managed Python directory name: {0}")]
NameError(String),
#[error(transparent)]
NameParseError(#[from] installation::PythonInstallationKeyError),
}
/// A collection of uv-managed Python installations installed on the current system.
#[derive(Debug, Clone)]
pub struct ManagedPythonInstallations {
/// The path to the top-level directory of the installed Python versions.
root: PathBuf,
}
impl ManagedPythonInstallations {
/// A directory for Python installations at `root`.
fn from_path(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
/// Lock the toolchains directory.
pub fn acquire_lock(&self) -> Result<LockedFile, Error> {
Ok(LockedFile::acquire(
self.root.join(".lock"),
self.root.user_display(),
)?)
}
/// Prefer, in order:
/// 1. The specific Python directory specified by the user, i.e., `UV_PYTHON_INSTALL_DIR`
/// 2. A directory in the system-appropriate user-level data directory, e.g., `~/.local/uv/python`
/// 3. A directory in the local data directory, e.g., `./.uv/python`
pub fn from_settings() -> Result<Self, Error> {
if let Some(install_dir) = std::env::var_os("UV_PYTHON_INSTALL_DIR") {
Ok(Self::from_path(install_dir))
} else {
Ok(Self::from_path(
StateStore::from_settings(None)?.bucket(StateBucket::ManagedPython),
))
}
}
/// Create a temporary Python installation directory.
pub fn temp() -> Result<Self, Error> {
Ok(Self::from_path(
StateStore::temp()?.bucket(StateBucket::ManagedPython),
))
}
/// Initialize the Python installation directory.
///
/// Ensures the directory is created.
pub fn init(self) -> Result<Self, Error> {
let root = &self.root;
// Support `toolchains` -> `python` migration transparently.
if !root.exists()
&& root
.parent()
.is_some_and(|parent| parent.join("toolchains").exists())
{
let deprecated = root.parent().unwrap().join("toolchains");
// Move the deprecated directory to the new location.
fs::rename(&deprecated, root)?;
// Create a link or junction to at the old location
uv_fs::replace_symlink(root, &deprecated)?;
} else {
fs::create_dir_all(root)?;
}
// Create the directory, if it doesn't exist.
fs::create_dir_all(root)?;
// Add a .gitignore.
match fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(root.join(".gitignore"))
{
Ok(mut file) => file.write_all(b"*")?,
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (),
Err(err) => return Err(err.into()),
}
Ok(self)
}
/// Iterate over each Python installation in this directory.
///
/// Pythons are sorted descending by name, such that we get deterministic
/// ordering across platforms. This also results in newer Python versions coming first,
/// but should not be relied on — instead the installations should be sorted later by
/// the parsed Python version.
pub fn find_all(
&self,
) -> Result<impl DoubleEndedIterator<Item = ManagedPythonInstallation>, Error> {
let dirs = match fs_err::read_dir(&self.root) {
Ok(installation_dirs) => {
// Collect sorted directory paths; `read_dir` is not stable across platforms
let directories: BTreeSet<_> = installation_dirs
.filter_map(|read_dir| match read_dir {
Ok(entry) => match entry.file_type() {
Ok(file_type) => file_type.is_dir().then_some(Ok(entry.path())),
Err(err) => Some(Err(err)),
},
Err(err) => Some(Err(err)),
})
.collect::<Result<_, std::io::Error>>()
.map_err(|err| Error::ReadError {
dir: self.root.clone(),
err,
})?;
directories
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => BTreeSet::default(),
Err(err) => {
return Err(Error::ReadError {
dir: self.root.clone(),
err,
})
}
};
Ok(dirs
.into_iter()
.filter_map(|path| {
ManagedPythonInstallation::new(path)
.inspect_err(|err| {
warn!("Ignoring malformed managed Python entry:\n {err}");
})
.ok()
})
.rev())
}
/// Iterate over Python installations that support the current platform.
pub fn find_matching_current_platform(
&self,
) -> Result<impl DoubleEndedIterator<Item = ManagedPythonInstallation>, Error> {
let platform_key = platform_key_from_env();
let iter = ManagedPythonInstallations::from_settings()?
.find_all()?
.filter(move |installation| {
installation
.path
.file_name()
.map(OsStr::to_string_lossy)
.is_some_and(|filename| filename.ends_with(&platform_key))
});
Ok(iter)
}
/// Iterate over managed Python installations that satisfy the requested version on this platform.
///
/// ## Errors
///
/// - The platform metadata cannot be read
/// - A directory for the installation cannot be read
pub fn find_version<'a>(
&self,
version: &'a PythonVersion,
) -> Result<impl DoubleEndedIterator<Item = ManagedPythonInstallation> + 'a, Error> {
Ok(self
.find_matching_current_platform()?
.filter(move |installation| {
installation
.path
.file_name()
.map(OsStr::to_string_lossy)
.is_some_and(|filename| filename.starts_with(&format!("cpython-{version}")))
}))
}
pub fn root(&self) -> &Path {
&self.root
}
}
static EXTERNALLY_MANAGED: &str = "[externally-managed]
Error=This Python installation is managed by uv and should not be modified.
";
/// A uv-managed Python installation on the current system..
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub struct ManagedPythonInstallation {
/// The path to the top-level directory of the installed Python.
path: PathBuf,
/// An install key for the Python version.
key: PythonInstallationKey,
}
impl ManagedPythonInstallation {
pub fn new(path: PathBuf) -> Result<Self, Error> {
let key = PythonInstallationKey::from_str(
path.file_name()
.ok_or(Error::NameError("name is empty".to_string()))?
.to_str()
.ok_or(Error::NameError("not a valid string".to_string()))?,
)?;
Ok(Self { path, key })
}
/// The path to this toolchain's Python executable.
pub fn executable(&self) -> PathBuf {
if cfg!(windows) {
self.path.join("install").join("python.exe")
} else if cfg!(unix) {
self.path.join("install").join("bin").join("python3")
} else {
unimplemented!("Only Windows and Unix systems are supported.")
}
}
/// The [`PythonVersion`] of the toolchain.
pub fn version(&self) -> PythonVersion {
self.key.version()
}
pub fn implementation(&self) -> &ImplementationName {
match self.key.implementation() {
LenientImplementationName::Known(implementation) => implementation,
LenientImplementationName::Unknown(_) => {
panic!("Managed Python installations should have a known implementation")
}
}
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn key(&self) -> &PythonInstallationKey {
&self.key
}
pub fn satisfies(&self, request: &PythonRequest) -> bool {
match request {
PythonRequest::File(path) => self.executable() == *path,
PythonRequest::Any => true,
PythonRequest::Directory(path) => self.path() == *path,
PythonRequest::ExecutableName(name) => self
.executable()
.file_name()
.is_some_and(|filename| filename.to_string_lossy() == *name),
PythonRequest::Implementation(implementation) => {
implementation == self.implementation()
}
PythonRequest::ImplementationVersion(implementation, version) => {
implementation == self.implementation() && version.matches_version(&self.version())
}
PythonRequest::Version(version) => version.matches_version(&self.version()),
PythonRequest::Key(request) => request.satisfied_by_key(self.key()),
}
}
/// Ensure the environment is marked as externally managed with the
/// standard `EXTERNALLY-MANAGED` file.
pub fn ensure_externally_managed(&self) -> Result<(), Error> {
// Construct the path to the `stdlib` directory.
let stdlib = if cfg!(windows) {
self.path.join("install").join("Lib")
} else {
self.path
.join("install")
.join("lib")
.join(format!("python{}", self.key.version().python_version()))
};
let file = stdlib.join("EXTERNALLY-MANAGED");
fs_err::write(file, EXTERNALLY_MANAGED)?;
Ok(())
}
}
/// Generate a platform portion of a key from the environment.
fn platform_key_from_env() -> String {
let os = Os::from_env();
let arch = Arch::from_env();
let libc = Libc::from_env();
format!("{os}-{arch}-{libc}").to_lowercase()
}
impl fmt::Display for ManagedPythonInstallation {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{}",
self.path
.file_name()
.unwrap_or(self.path.as_os_str())
.to_string_lossy()
)
}
}

View file

@ -0,0 +1,195 @@
use std::fmt::Display;
use std::ops::Deref;
use std::{fmt, str::FromStr};
use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("Unknown operating system: {0}")]
UnknownOs(String),
#[error("Unknown architecture: {0}")]
UnknownArch(String),
#[error("Unknown libc environment: {0}")]
UnknownLibc(String),
}
#[derive(Debug, Eq, PartialEq, Clone, Copy)]
pub struct Arch(pub(crate) target_lexicon::Architecture);
#[derive(Debug, Eq, PartialEq, Clone, Copy)]
pub struct Os(pub(crate) target_lexicon::OperatingSystem);
#[derive(Debug, Eq, PartialEq, Clone, Copy)]
pub enum Libc {
Some(target_lexicon::Environment),
None,
}
impl Libc {
pub(crate) fn from_env() -> Self {
match std::env::consts::OS {
// TODO(zanieb): On Linux, we use the uv target host to determine the libc variant
// but we should only use this as a fallback and should instead inspect the
// machine's `/bin/sh` (or similar).
"linux" => Self::Some(target_lexicon::Environment::Gnu),
"windows" | "macos" => Self::None,
// Use `None` on platforms without explicit support.
_ => Self::None,
}
}
}
impl FromStr for Libc {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"gnu" => Ok(Self::Some(target_lexicon::Environment::Gnu)),
"musl" => Ok(Self::Some(target_lexicon::Environment::Musl)),
"none" => Ok(Self::None),
_ => Err(Error::UnknownLibc(s.to_string())),
}
}
}
impl Os {
pub fn from_env() -> Self {
Self(target_lexicon::HOST.operating_system)
}
}
impl Arch {
pub fn from_env() -> Self {
Self(target_lexicon::HOST.architecture)
}
}
impl Display for Libc {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Some(env) => write!(f, "{env}"),
Self::None => write!(f, "none"),
}
}
}
impl Display for Os {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &**self {
target_lexicon::OperatingSystem::Darwin => write!(f, "macos"),
inner => write!(f, "{inner}"),
}
}
}
impl Display for Arch {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &**self {
target_lexicon::Architecture::X86_32(target_lexicon::X86_32Architecture::I686) => {
write!(f, "x86")
}
inner => write!(f, "{inner}"),
}
}
}
impl FromStr for Os {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let inner = match s {
"macos" => target_lexicon::OperatingSystem::Darwin,
_ => target_lexicon::OperatingSystem::from_str(s)
.map_err(|()| Error::UnknownOs(s.to_string()))?,
};
if matches!(inner, target_lexicon::OperatingSystem::Unknown) {
return Err(Error::UnknownOs(s.to_string()));
}
Ok(Self(inner))
}
}
impl FromStr for Arch {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let inner = match s {
// Allow users to specify "x86" as a shorthand for the "i686" variant, they should not need
// to specify the exact architecture and this variant is what we have downloads for.
"x86" => target_lexicon::Architecture::X86_32(target_lexicon::X86_32Architecture::I686),
_ => target_lexicon::Architecture::from_str(s)
.map_err(|()| Error::UnknownArch(s.to_string()))?,
};
if matches!(inner, target_lexicon::Architecture::Unknown) {
return Err(Error::UnknownArch(s.to_string()));
}
Ok(Self(inner))
}
}
impl Deref for Arch {
type Target = target_lexicon::Architecture;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Deref for Os {
type Target = target_lexicon::OperatingSystem;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl From<&platform_tags::Arch> for Arch {
fn from(value: &platform_tags::Arch) -> Self {
match value {
platform_tags::Arch::Aarch64 => Self(target_lexicon::Architecture::Aarch64(
target_lexicon::Aarch64Architecture::Aarch64,
)),
platform_tags::Arch::Armv6L => Self(target_lexicon::Architecture::Arm(
target_lexicon::ArmArchitecture::Armv6,
)),
platform_tags::Arch::Armv7L => Self(target_lexicon::Architecture::Arm(
target_lexicon::ArmArchitecture::Armv7,
)),
platform_tags::Arch::S390X => Self(target_lexicon::Architecture::S390x),
platform_tags::Arch::Powerpc64 => Self(target_lexicon::Architecture::Powerpc64),
platform_tags::Arch::Powerpc64Le => Self(target_lexicon::Architecture::Powerpc64le),
platform_tags::Arch::X86 => Self(target_lexicon::Architecture::X86_32(
target_lexicon::X86_32Architecture::I686,
)),
platform_tags::Arch::X86_64 => Self(target_lexicon::Architecture::X86_64),
}
}
}
impl From<&platform_tags::Os> for Libc {
fn from(value: &platform_tags::Os) -> Self {
match value {
platform_tags::Os::Manylinux { .. } => Self::Some(target_lexicon::Environment::Gnu),
platform_tags::Os::Musllinux { .. } => Self::Some(target_lexicon::Environment::Musl),
_ => Self::None,
}
}
}
impl From<&platform_tags::Os> for Os {
fn from(value: &platform_tags::Os) -> Self {
match value {
platform_tags::Os::Dragonfly { .. } => Self(target_lexicon::OperatingSystem::Dragonfly),
platform_tags::Os::FreeBsd { .. } => Self(target_lexicon::OperatingSystem::Freebsd),
platform_tags::Os::Haiku { .. } => Self(target_lexicon::OperatingSystem::Haiku),
platform_tags::Os::Illumos { .. } => Self(target_lexicon::OperatingSystem::Illumos),
platform_tags::Os::Macos { .. } => Self(target_lexicon::OperatingSystem::Darwin),
platform_tags::Os::Manylinux { .. } | platform_tags::Os::Musllinux { .. } => {
Self(target_lexicon::OperatingSystem::Linux)
}
platform_tags::Os::NetBsd { .. } => Self(target_lexicon::OperatingSystem::Netbsd),
platform_tags::Os::OpenBsd { .. } => Self(target_lexicon::OperatingSystem::Openbsd),
platform_tags::Os::Windows => Self(target_lexicon::OperatingSystem::Windows),
}
}
}

View file

@ -0,0 +1,21 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum PointerSize {
/// 32-bit architecture.
#[serde(rename = "32")]
_32,
/// 64-bit architecture.
#[serde(rename = "64")]
_64,
}
impl PointerSize {
pub const fn is_32(self) -> bool {
matches!(self, Self::_32)
}
pub const fn is_64(self) -> bool {
matches!(self, Self::_64)
}
}

View file

@ -0,0 +1,43 @@
use std::path::{Path, PathBuf};
use pypi_types::Scheme;
/// A `--prefix` directory into which packages can be installed, separate from a virtual environment
/// or system Python interpreter.
#[derive(Debug, Clone)]
pub struct Prefix(PathBuf);
impl Prefix {
/// Return the [`Scheme`] for the `--prefix` directory.
pub fn scheme(&self, virtualenv: &Scheme) -> Scheme {
Scheme {
purelib: self.0.join(&virtualenv.purelib),
platlib: self.0.join(&virtualenv.platlib),
scripts: self.0.join(&virtualenv.scripts),
data: self.0.join(&virtualenv.data),
include: self.0.join(&virtualenv.include),
}
}
/// Return an iterator over the `site-packages` directories inside the environment.
pub fn site_packages(&self, virtualenv: &Scheme) -> impl Iterator<Item = PathBuf> {
std::iter::once(self.0.join(&virtualenv.purelib))
}
/// Initialize the `--prefix` directory.
pub fn init(&self) -> std::io::Result<()> {
fs_err::create_dir_all(&self.0)?;
Ok(())
}
/// Return the path to the `--prefix` directory.
pub fn root(&self) -> &Path {
&self.0
}
}
impl From<PathBuf> for Prefix {
fn from(path: PathBuf) -> Self {
Self(path)
}
}

View file

@ -0,0 +1,94 @@
use std::io;
use std::path::PathBuf;
use std::process::{Command, ExitStatus};
use once_cell::sync::Lazy;
use regex::Regex;
use thiserror::Error;
use tracing::info_span;
#[derive(Debug, Clone)]
pub(crate) struct PyListPath {
pub(crate) major: u8,
pub(crate) minor: u8,
pub(crate) executable_path: PathBuf,
}
/// An error was encountered when using the `py` launcher on Windows.
#[derive(Error, Debug)]
pub enum Error {
#[error("{message} with {exit_code}\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---")]
StatusCode {
message: String,
exit_code: ExitStatus,
stdout: String,
stderr: String,
},
#[error("Failed to run `py --list-paths` to find Python installations.")]
Io(#[source] io::Error),
#[error("The `py` launcher could not be found.")]
NotFound,
}
/// ```text
/// -V:3.12 C:\Users\Ferris\AppData\Local\Programs\Python\Python312\python.exe
/// -V:3.8 C:\Users\Ferris\AppData\Local\Programs\Python\Python38\python.exe
/// ```
static PY_LIST_PATHS: Lazy<Regex> = Lazy::new(|| {
// Without the `R` flag, paths have trailing \r
Regex::new(r"(?mR)^ -(?:V:)?(\d).(\d+)-?(?:arm)?\d*\s*\*?\s*(.*)$").unwrap()
});
/// Use the `py` launcher to find installed Python versions.
///
/// Calls `py --list-paths`.
pub(crate) fn py_list_paths() -> Result<Vec<PyListPath>, Error> {
// konstin: The command takes 8ms on my machine.
let output = info_span!("py_list_paths")
.in_scope(|| Command::new("py").arg("--list-paths").output())
.map_err(|err| {
if err.kind() == std::io::ErrorKind::NotFound {
Error::NotFound
} else {
Error::Io(err)
}
})?;
// `py` sometimes prints "Installed Pythons found by py Launcher for Windows" to stderr which we ignore.
if !output.status.success() {
return Err(Error::StatusCode {
message: format!(
"Running `py --list-paths` failed with status {}",
output.status
),
exit_code: output.status,
stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
});
}
// Find the first python of the version we want in the list
let stdout = String::from_utf8(output.stdout).map_err(|err| Error::StatusCode {
message: format!("The stdout of `py --list-paths` isn't UTF-8 encoded: {err}"),
exit_code: output.status,
stdout: String::from_utf8_lossy(err.as_bytes()).trim().to_string(),
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
})?;
Ok(PY_LIST_PATHS
.captures_iter(&stdout)
.filter_map(|captures| {
let (_, [major, minor, path]) = captures.extract();
if let (Some(major), Some(minor)) = (major.parse::<u8>().ok(), minor.parse::<u8>().ok())
{
Some(PyListPath {
major,
minor,
executable_path: PathBuf::from(path),
})
} else {
None
}
})
.collect())
}

View file

@ -0,0 +1,210 @@
use std::fmt::{Display, Formatter};
use std::ops::Deref;
use std::str::FromStr;
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, StringVersion};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PythonVersion(StringVersion);
impl Deref for PythonVersion {
type Target = StringVersion;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl FromStr for PythonVersion {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let version = StringVersion::from_str(s)
.map_err(|err| format!("Python version `{s}` could not be parsed: {err}"))?;
if version.is_dev() {
return Err(format!("Python version `{s}` is a development release"));
}
if version.is_local() {
return Err(format!("Python version `{s}` is a local version"));
}
if version.epoch() != 0 {
return Err(format!("Python version `{s}` has a non-zero epoch"));
}
if version.version < Version::new([3, 7]) {
return Err(format!("Python version `{s}` must be >= 3.7"));
}
if version.version >= Version::new([4, 0]) {
return Err(format!("Python version `{s}` must be < 4.0"));
}
Ok(Self(version))
}
}
#[cfg(feature = "schemars")]
impl schemars::JsonSchema for PythonVersion {
fn schema_name() -> String {
String::from("PythonVersion")
}
fn json_schema(_gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema {
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::String.into()),
string: Some(Box::new(schemars::schema::StringValidation {
pattern: Some(r"^3\.\d+(\.\d+)?$".to_string()),
..schemars::schema::StringValidation::default()
})),
metadata: Some(Box::new(schemars::schema::Metadata {
description: Some("A Python version specifier, e.g. `3.7` or `3.8.0`.".to_string()),
..schemars::schema::Metadata::default()
})),
..schemars::schema::SchemaObject::default()
}
.into()
}
}
impl<'de> serde::Deserialize<'de> for PythonVersion {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
PythonVersion::from_str(&s).map_err(serde::de::Error::custom)
}
}
impl Display for PythonVersion {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
Display::fmt(&self.0, f)
}
}
impl PythonVersion {
/// Return a [`MarkerEnvironment`] compatible with the given [`PythonVersion`], based on
/// a base [`MarkerEnvironment`].
///
/// The returned [`MarkerEnvironment`] will preserve the base environment's platform markers,
/// but override its Python version markers.
pub fn markers(self, base: &MarkerEnvironment) -> MarkerEnvironment {
let mut markers = base.clone();
// Ex) `implementation_version == "3.12.0"`
if markers.implementation_name() == "cpython" {
let python_full_version = self.python_full_version();
markers = markers.with_implementation_version(StringVersion {
// Retain the verbatim representation, provided by the user.
string: self.0.to_string(),
version: python_full_version,
});
}
// Ex) `python_full_version == "3.12.0"`
let python_full_version = self.python_full_version();
markers = markers.with_python_full_version(StringVersion {
// Retain the verbatim representation, provided by the user.
string: self.0.to_string(),
version: python_full_version,
});
// Ex) `python_version == "3.12"`
let python_version = self.python_version();
markers = markers.with_python_version(StringVersion {
string: python_version.to_string(),
version: python_version,
});
markers
}
/// Return the `python_version` marker corresponding to this Python version.
///
/// This should include exactly a major and minor version, but no patch version.
///
/// Ex) `python_version == "3.12"`
pub fn python_version(&self) -> Version {
let major = self.release().first().copied().unwrap_or(0);
let minor = self.release().get(1).copied().unwrap_or(0);
Version::new([major, minor])
}
/// Return the `python_full_version` marker corresponding to this Python version.
///
/// This should include exactly a major, minor, and patch version (even if it's zero), along
/// with any pre-release or post-release information.
///
/// Ex) `python_full_version == "3.12.0b1"`
pub fn python_full_version(&self) -> Version {
let major = self.release().first().copied().unwrap_or(0);
let minor = self.release().get(1).copied().unwrap_or(0);
let patch = self.release().get(2).copied().unwrap_or(0);
Version::new([major, minor, patch])
.with_pre(self.0.pre())
.with_post(self.0.post())
}
/// Return the full parsed Python version.
pub fn version(&self) -> &Version {
&self.0.version
}
/// Return the major version of this Python version.
pub fn major(&self) -> u8 {
u8::try_from(self.0.release().first().copied().unwrap_or(0)).expect("invalid major version")
}
/// Return the minor version of this Python version.
pub fn minor(&self) -> u8 {
u8::try_from(self.0.release().get(1).copied().unwrap_or(0)).expect("invalid minor version")
}
/// Return the patch version of this Python version, if set.
pub fn patch(&self) -> Option<u8> {
self.0
.release()
.get(2)
.copied()
.map(|patch| u8::try_from(patch).expect("invalid patch version"))
}
/// Returns a copy of the Python version without the patch version
#[must_use]
pub fn without_patch(&self) -> Self {
Self::from_str(format!("{}.{}", self.major(), self.minor()).as_str())
.expect("dropping a patch should always be valid")
}
}
#[cfg(test)]
mod tests {
use std::str::FromStr;
use pep440_rs::{PreRelease, PreReleaseKind, Version};
use crate::PythonVersion;
#[test]
fn python_markers() {
let version = PythonVersion::from_str("3.11.0").expect("valid python version");
assert_eq!(version.python_version(), Version::new([3, 11]));
assert_eq!(version.python_version().to_string(), "3.11");
assert_eq!(version.python_full_version(), Version::new([3, 11, 0]));
assert_eq!(version.python_full_version().to_string(), "3.11.0");
let version = PythonVersion::from_str("3.11").expect("valid python version");
assert_eq!(version.python_version(), Version::new([3, 11]));
assert_eq!(version.python_version().to_string(), "3.11");
assert_eq!(version.python_full_version(), Version::new([3, 11, 0]));
assert_eq!(version.python_full_version().to_string(), "3.11.0");
let version = PythonVersion::from_str("3.11.8a1").expect("valid python version");
assert_eq!(version.python_version(), Version::new([3, 11]));
assert_eq!(version.python_version().to_string(), "3.11");
assert_eq!(
version.python_full_version(),
Version::new([3, 11, 8]).with_pre(Some(PreRelease {
kind: PreReleaseKind::Alpha,
number: 1
}))
);
assert_eq!(version.python_full_version().to_string(), "3.11.8a1");
}
}

View file

@ -0,0 +1,43 @@
use std::path::{Path, PathBuf};
use pypi_types::Scheme;
/// A `--target` directory into which packages can be installed, separate from a virtual environment
/// or system Python interpreter.
#[derive(Debug, Clone)]
pub struct Target(PathBuf);
impl Target {
/// Return the [`Scheme`] for the `--target` directory.
pub fn scheme(&self) -> Scheme {
Scheme {
purelib: self.0.clone(),
platlib: self.0.clone(),
scripts: self.0.join("bin"),
data: self.0.clone(),
include: self.0.join("include"),
}
}
/// Return an iterator over the `site-packages` directories inside the environment.
pub fn site_packages(&self) -> impl Iterator<Item = &Path> {
std::iter::once(self.0.as_path())
}
/// Initialize the `--target` directory.
pub fn init(&self) -> std::io::Result<()> {
fs_err::create_dir_all(&self.0)?;
Ok(())
}
/// Return the path to the `--target` directory.
pub fn root(&self) -> &Path {
&self.0
}
}
impl From<PathBuf> for Target {
fn from(path: PathBuf) -> Self {
Self(path)
}
}

View file

@ -0,0 +1,67 @@
use fs_err as fs;
use std::{io, path::PathBuf};
use tracing::debug;
use crate::PythonRequest;
/// Read [`PythonRequest`]s from a version file, if present.
///
/// Prefers `.python-versions` then `.python-version`.
/// If only one Python version is desired, use [`request_from_version_files`] which prefers the `.python-version` file.
pub async fn requests_from_version_file() -> Result<Option<Vec<PythonRequest>>, io::Error> {
if let Some(versions) = read_versions_file().await? {
Ok(Some(
versions
.into_iter()
.map(|version| PythonRequest::parse(&version))
.collect(),
))
} else if let Some(version) = read_version_file().await? {
Ok(Some(vec![PythonRequest::parse(&version)]))
} else {
Ok(None)
}
}
/// Read a [`PythonRequest`] from a version file, if present.
///
/// Prefers `.python-version` then the first entry of `.python-versions`.
/// If multiple Python versions are desired, use [`requests_from_version_files`] instead.
pub async fn request_from_version_file() -> Result<Option<PythonRequest>, io::Error> {
if let Some(version) = read_version_file().await? {
Ok(Some(PythonRequest::parse(&version)))
} else if let Some(versions) = read_versions_file().await? {
Ok(versions
.into_iter()
.next()
.inspect(|_| debug!("Using the first version from `.python-versions`"))
.map(|version| PythonRequest::parse(&version)))
} else {
Ok(None)
}
}
async fn read_versions_file() -> Result<Option<Vec<String>>, io::Error> {
if !PathBuf::from(".python-versions").try_exists()? {
return Ok(None);
}
debug!("Reading requests from `.python-versions`");
let lines: Vec<String> = fs::tokio::read_to_string(".python-versions")
.await?
.lines()
.map(ToString::to_string)
.collect();
Ok(Some(lines))
}
async fn read_version_file() -> Result<Option<String>, io::Error> {
if !PathBuf::from(".python-version").try_exists()? {
return Ok(None);
}
debug!("Reading requests from `.python-version`");
Ok(fs::tokio::read_to_string(".python-version")
.await?
.lines()
.next()
.map(ToString::to_string))
}

View file

@ -0,0 +1,172 @@
use std::{
env, io,
path::{Path, PathBuf},
};
use fs_err as fs;
use pypi_types::Scheme;
use thiserror::Error;
/// The layout of a virtual environment.
#[derive(Debug)]
pub struct VirtualEnvironment {
/// The absolute path to the root of the virtualenv, e.g., `/path/to/.venv`.
pub root: PathBuf,
/// The path to the Python interpreter inside the virtualenv, e.g., `.venv/bin/python`
/// (Unix, Python 3.11).
pub executable: PathBuf,
/// The [`Scheme`] paths for the virtualenv, as returned by (e.g.) `sysconfig.get_paths()`.
pub scheme: Scheme,
}
/// A parsed `pyvenv.cfg`
#[derive(Debug, Clone)]
pub struct PyVenvConfiguration {
/// If the `virtualenv` package was used to create the virtual environment.
pub(crate) virtualenv: bool,
/// If the `uv` package was used to create the virtual environment.
pub(crate) uv: bool,
}
#[derive(Debug, Error)]
pub enum Error {
#[error("Broken virtualenv `{0}`: `pyvenv.cfg` is missing")]
MissingPyVenvCfg(PathBuf),
#[error("Broken virtualenv `{0}`: `pyvenv.cfg` could not be parsed")]
ParsePyVenvCfg(PathBuf, #[source] io::Error),
#[error(transparent)]
IO(#[from] io::Error),
}
/// Locate an active virtual environment by inspecting environment variables.
///
/// Supports `VIRTUAL_ENV`.
pub(crate) fn virtualenv_from_env() -> Option<PathBuf> {
if let Some(dir) = env::var_os("VIRTUAL_ENV").filter(|value| !value.is_empty()) {
return Some(PathBuf::from(dir));
}
None
}
/// Locate an active conda environment by inspecting environment variables.
///
/// Supports `CONDA_PREFIX`.
pub(crate) fn conda_prefix_from_env() -> Option<PathBuf> {
if let Some(dir) = env::var_os("CONDA_PREFIX").filter(|value| !value.is_empty()) {
return Some(PathBuf::from(dir));
}
None
}
/// Locate a virtual environment by searching the file system.
///
/// Searches for a `.venv` directory in the current or any parent directory. If the current
/// directory is itself a virtual environment (or a subdirectory of a virtual environment), the
/// containing virtual environment is returned.
pub(crate) fn virtualenv_from_working_dir() -> Result<Option<PathBuf>, Error> {
let current_dir = crate::current_dir()?;
for dir in current_dir.ancestors() {
// If we're _within_ a virtualenv, return it.
if dir.join("pyvenv.cfg").is_file() {
return Ok(Some(dir.to_path_buf()));
}
// Otherwise, search for a `.venv` directory.
let dot_venv = dir.join(".venv");
if dot_venv.is_dir() {
if !dot_venv.join("pyvenv.cfg").is_file() {
return Err(Error::MissingPyVenvCfg(dot_venv));
}
return Ok(Some(dot_venv));
}
}
Ok(None)
}
/// Returns the path to the `python` executable inside a virtual environment.
pub(crate) fn virtualenv_python_executable(venv: impl AsRef<Path>) -> PathBuf {
let venv = venv.as_ref();
if cfg!(windows) {
// Search for `python.exe` in the `Scripts` directory.
let default_executable = venv.join("Scripts").join("python.exe");
if default_executable.exists() {
return default_executable;
}
// Apparently, Python installed via msys2 on Windows _might_ produce a POSIX-like layout.
// See: https://github.com/PyO3/maturin/issues/1108
let executable = venv.join("bin").join("python.exe");
if executable.exists() {
return executable;
}
// Fallback for Conda environments.
let executable = venv.join("python.exe");
if executable.exists() {
return executable;
}
// If none of these exist, return the standard location
default_executable
} else {
// Check for both `python3` over `python`, preferring the more specific one
let default_executable = venv.join("bin").join("python3");
if default_executable.exists() {
return default_executable;
}
let executable = venv.join("bin").join("python");
if executable.exists() {
return executable;
}
// If none of these exist, return the standard location
default_executable
}
}
impl PyVenvConfiguration {
/// Parse a `pyvenv.cfg` file into a [`PyVenvConfiguration`].
pub fn parse(cfg: impl AsRef<Path>) -> Result<Self, Error> {
let mut virtualenv = false;
let mut uv = false;
// Per https://snarky.ca/how-virtual-environments-work/, the `pyvenv.cfg` file is not a
// valid INI file, and is instead expected to be parsed by partitioning each line on the
// first equals sign.
let content = fs::read_to_string(&cfg)
.map_err(|err| Error::ParsePyVenvCfg(cfg.as_ref().to_path_buf(), err))?;
for line in content.lines() {
let Some((key, _value)) = line.split_once('=') else {
continue;
};
match key.trim() {
"virtualenv" => {
virtualenv = true;
}
"uv" => {
uv = true;
}
_ => {}
}
}
Ok(Self { virtualenv, uv })
}
/// Returns true if the virtual environment was created with the `virtualenv` package.
pub fn is_virtualenv(&self) -> bool {
self.virtualenv
}
/// Returns true if the virtual environment was created with the `uv` package.
pub fn is_uv(&self) -> bool {
self.uv
}
}