Add support for listing system toolchains (#4172)

Includes system interpreters in `uv toolchain list`.

This includes a refactor of `find_toolchain` to support iterating over
all toolchains
that match a request rather than ending earlier.
This commit is contained in:
Zanie Blue 2024-06-13 11:25:30 -04:00 committed by GitHub
parent 39b8c06842
commit 89daa51dbe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 378 additions and 206 deletions

View file

@ -452,6 +452,137 @@ fn should_stop_discovery(err: &Error) -> bool {
}
}
fn find_toolchain_at_file(path: &PathBuf, cache: &Cache) -> Result<ToolchainResult, Error> {
if !path.try_exists()? {
return Ok(ToolchainResult::Err(ToolchainNotFound::FileNotFound(
path.clone(),
)));
}
Ok(ToolchainResult::Ok(Toolchain {
source: ToolchainSource::ProvidedPath,
interpreter: Interpreter::query(path, cache)?,
}))
}
fn find_toolchain_at_directory(path: &PathBuf, cache: &Cache) -> Result<ToolchainResult, Error> {
if !path.try_exists()? {
return Ok(ToolchainResult::Err(ToolchainNotFound::FileNotFound(
path.clone(),
)));
}
let executable = virtualenv_python_executable(path);
if !executable.try_exists()? {
return Ok(ToolchainResult::Err(
ToolchainNotFound::ExecutableNotFoundInDirectory(path.clone(), executable),
));
}
Ok(ToolchainResult::Ok(Toolchain {
source: ToolchainSource::ProvidedPath,
interpreter: Interpreter::query(executable, cache)?,
}))
}
fn find_toolchain_with_executable_name(
name: &str,
cache: &Cache,
) -> Result<ToolchainResult, Error> {
let Some(executable) = which(name).ok() else {
return Ok(ToolchainResult::Err(
ToolchainNotFound::ExecutableNotFoundInSearchPath(name.to_string()),
));
};
Ok(ToolchainResult::Ok(Toolchain {
source: ToolchainSource::SearchPath,
interpreter: Interpreter::query(executable, cache)?,
}))
}
/// Iterate over all toolchains that satisfy the given request.
pub fn find_toolchains<'a>(
request: &'a ToolchainRequest,
system: SystemPython,
sources: &'a ToolchainSources,
cache: &'a Cache,
) -> Box<dyn Iterator<Item = Result<ToolchainResult, Error>> + 'a> {
match request {
ToolchainRequest::File(path) => Box::new(std::iter::once({
if sources.contains(ToolchainSource::ProvidedPath) {
debug!("Checking for Python interpreter at {request}");
find_toolchain_at_file(path, cache)
} else {
Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::ProvidedPath,
sources.clone(),
))
}
})),
ToolchainRequest::Directory(path) => Box::new(std::iter::once({
debug!("Checking for Python interpreter in {request}");
if sources.contains(ToolchainSource::ProvidedPath) {
debug!("Checking for Python interpreter at {request}");
find_toolchain_at_directory(path, cache)
} else {
Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::ProvidedPath,
sources.clone(),
))
}
})),
ToolchainRequest::ExecutableName(name) => Box::new(std::iter::once({
debug!("Searching for Python interpreter with {request}");
if sources.contains(ToolchainSource::SearchPath) {
debug!("Checking for Python interpreter at {request}");
find_toolchain_with_executable_name(name, cache)
} else {
Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::SearchPath,
sources.clone(),
))
}
})),
ToolchainRequest::Any => Box::new({
debug!("Searching for Python interpreter in {sources}");
python_interpreters(None, None, system, sources, cache)
.map(|result| result.map(Toolchain::from_tuple).map(ToolchainResult::Ok))
}),
ToolchainRequest::Version(version) => Box::new({
debug!("Searching for {request} in {sources}");
python_interpreters(Some(version), None, system, sources, cache)
.filter(|result| match result {
Err(_) => true,
Ok((_source, interpreter)) => version.matches_interpreter(interpreter),
})
.map(|result| result.map(Toolchain::from_tuple).map(ToolchainResult::Ok))
}),
ToolchainRequest::Implementation(implementation) => Box::new({
debug!("Searching for a {request} interpreter in {sources}");
python_interpreters(None, Some(implementation), system, sources, cache)
.filter(|result| match result {
Err(_) => true,
Ok((_source, interpreter)) => {
interpreter.implementation_name() == implementation.as_str()
}
})
.map(|result| result.map(Toolchain::from_tuple).map(ToolchainResult::Ok))
}),
ToolchainRequest::ImplementationVersion(implementation, version) => Box::new({
debug!("Searching for {request} in {sources}");
python_interpreters(Some(version), Some(implementation), system, sources, cache)
.filter(|result| match result {
Err(_) => true,
Ok((_source, interpreter)) => {
version.matches_interpreter(interpreter)
&& interpreter.implementation_name() == implementation.as_str()
}
})
.map(|result| result.map(Toolchain::from_tuple).map(ToolchainResult::Ok))
}),
}
}
/// Find a toolchain that satisfies the given request.
///
/// If an error is encountered while locating or inspecting a candidate toolchain,
@ -462,175 +593,38 @@ pub(crate) fn find_toolchain(
sources: &ToolchainSources,
cache: &Cache,
) -> Result<ToolchainResult, Error> {
let result = match request {
ToolchainRequest::File(path) => {
debug!("Checking for Python interpreter at {request}");
if !sources.contains(ToolchainSource::ProvidedPath) {
return Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::ProvidedPath,
let mut toolchains = find_toolchains(request, system, sources, cache);
if let Some(result) = toolchains.find(|result| {
// Return the first critical discovery error or toolchain result
result.as_ref().err().map_or(true, should_stop_discovery)
}) {
result
} else {
let err = match request {
ToolchainRequest::Implementation(implementation) => {
ToolchainNotFound::NoMatchingImplementation(sources.clone(), *implementation)
}
ToolchainRequest::ImplementationVersion(implementation, version) => {
ToolchainNotFound::NoMatchingImplementationVersion(
sources.clone(),
));
*implementation,
version.clone(),
)
}
if !path.try_exists()? {
return Ok(ToolchainResult::Err(ToolchainNotFound::FileNotFound(
path.clone(),
)));
ToolchainRequest::Version(version) => {
ToolchainNotFound::NoMatchingVersion(sources.clone(), version.clone())
}
Toolchain {
source: ToolchainSource::ProvidedPath,
interpreter: Interpreter::query(path, cache)?,
// TODO(zanieb): As currently implemented, these are unreachable as they are handled in `find_toolchains`
// We should avoid this duplication
ToolchainRequest::Directory(path) => ToolchainNotFound::DirectoryNotFound(path.clone()),
ToolchainRequest::File(path) => ToolchainNotFound::FileNotFound(path.clone()),
ToolchainRequest::ExecutableName(name) => {
ToolchainNotFound::ExecutableNotFoundInSearchPath(name.clone())
}
}
ToolchainRequest::Directory(path) => {
debug!("Checking for Python interpreter in {request}");
if !sources.contains(ToolchainSource::ProvidedPath) {
return Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::ProvidedPath,
sources.clone(),
));
}
if !path.try_exists()? {
return Ok(ToolchainResult::Err(ToolchainNotFound::FileNotFound(
path.clone(),
)));
}
let executable = virtualenv_python_executable(path);
if !executable.try_exists()? {
return Ok(ToolchainResult::Err(
ToolchainNotFound::ExecutableNotFoundInDirectory(path.clone(), executable),
));
}
Toolchain {
source: ToolchainSource::ProvidedPath,
interpreter: Interpreter::query(executable, cache)?,
}
}
ToolchainRequest::ExecutableName(name) => {
debug!("Searching for Python interpreter with {request}");
if !sources.contains(ToolchainSource::SearchPath) {
return Err(Error::SourceNotSelected(
request.clone(),
ToolchainSource::SearchPath,
sources.clone(),
));
}
let Some(executable) = which(name).ok() else {
return Ok(ToolchainResult::Err(
ToolchainNotFound::ExecutableNotFoundInSearchPath(name.clone()),
));
};
Toolchain {
source: ToolchainSource::SearchPath,
interpreter: Interpreter::query(executable, cache)?,
}
}
ToolchainRequest::Implementation(implementation) => {
debug!("Searching for a {request} interpreter in {sources}");
let Some((source, interpreter)) =
python_interpreters(None, Some(implementation), system, sources, cache)
.find(|result| {
match result {
// Return the first critical error or matching interpreter
Err(err) => should_stop_discovery(err),
Ok((_source, interpreter)) => {
interpreter.implementation_name() == implementation.as_str()
}
}
})
.transpose()?
else {
return Ok(ToolchainResult::Err(
ToolchainNotFound::NoMatchingImplementation(sources.clone(), *implementation),
));
};
Toolchain {
source,
interpreter,
}
}
ToolchainRequest::ImplementationVersion(implementation, version) => {
debug!("Searching for {request} in {sources}");
let Some((source, interpreter)) =
python_interpreters(Some(version), Some(implementation), system, sources, cache)
.find(|result| {
match result {
// Return the first critical error or matching interpreter
Err(err) => should_stop_discovery(err),
Ok((_source, interpreter)) => {
version.matches_interpreter(interpreter)
&& interpreter.implementation_name() == implementation.as_str()
}
}
})
.transpose()?
else {
// TODO(zanieb): Peek if there are any interpreters with the requested implementation
// to improve the error message e.g. using `NoMatchingImplementation` instead
return Ok(ToolchainResult::Err(
ToolchainNotFound::NoMatchingImplementationVersion(
sources.clone(),
*implementation,
version.clone(),
),
));
};
Toolchain {
source,
interpreter,
}
}
ToolchainRequest::Any => {
debug!("Searching for Python interpreter in {sources}");
let Some((source, interpreter)) =
python_interpreters(None, None, system, sources, cache)
.find(|result| {
match result {
// Return the first critical error or interpreter
Err(err) => should_stop_discovery(err),
Ok(_) => true,
}
})
.transpose()?
else {
return Ok(ToolchainResult::Err(
ToolchainNotFound::NoPythonInstallation(sources.clone(), None),
));
};
Toolchain {
source,
interpreter,
}
}
ToolchainRequest::Version(version) => {
debug!("Searching for {request} in {sources}");
let Some((source, interpreter)) =
python_interpreters(Some(version), None, system, sources, cache)
.find(|result| {
match result {
// Return the first critical error or matching interpreter
Err(err) => should_stop_discovery(err),
Ok((_source, interpreter)) => version.matches_interpreter(interpreter),
}
})
.transpose()?
else {
let err = if matches!(version, VersionRequest::Any) {
ToolchainNotFound::NoPythonInstallation(sources.clone(), Some(version.clone()))
} else {
ToolchainNotFound::NoMatchingVersion(sources.clone(), version.clone())
};
return Ok(ToolchainResult::Err(err));
};
Toolchain {
source,
interpreter,
}
}
};
Ok(ToolchainResult::Ok(result))
ToolchainRequest::Any => ToolchainNotFound::NoPythonInstallation(sources.clone(), None),
};
Ok(ToolchainResult::Err(err))
}
}
/// Find the default Python toolchain on the system.

View file

@ -279,6 +279,10 @@ impl PythonDownload {
)
}
pub fn os(&self) -> &Os {
&self.os
}
pub fn sha256(&self) -> Option<&str> {
self.sha256
}

View file

@ -18,7 +18,7 @@ pub enum ImplementationName {
}
#[derive(Debug, Eq, PartialEq, Clone)]
pub(crate) enum LenientImplementationName {
pub enum LenientImplementationName {
Known(ImplementationName),
Unknown(String),
}

View file

@ -2,8 +2,8 @@
use thiserror::Error;
pub use crate::discovery::{
Error as DiscoveryError, SystemPython, ToolchainNotFound, ToolchainRequest, ToolchainSource,
ToolchainSources, VersionRequest,
find_toolchains, Error as DiscoveryError, SystemPython, ToolchainNotFound, ToolchainRequest,
ToolchainSource, ToolchainSources, VersionRequest,
};
pub use crate::environment::PythonEnvironment;
pub use crate::interpreter::Interpreter;

View file

@ -127,3 +127,54 @@ impl Deref for Os {
&self.0
}
}
impl From<&platform_tags::Arch> for Arch {
fn from(value: &platform_tags::Arch) -> Self {
match value {
platform_tags::Arch::Aarch64 => Self(target_lexicon::Architecture::Aarch64(
target_lexicon::Aarch64Architecture::Aarch64,
)),
platform_tags::Arch::Armv6L => Self(target_lexicon::Architecture::Arm(
target_lexicon::ArmArchitecture::Armv6,
)),
platform_tags::Arch::Armv7L => Self(target_lexicon::Architecture::Arm(
target_lexicon::ArmArchitecture::Armv7,
)),
platform_tags::Arch::S390X => Self(target_lexicon::Architecture::S390x),
platform_tags::Arch::Powerpc64 => Self(target_lexicon::Architecture::Powerpc64),
platform_tags::Arch::Powerpc64Le => Self(target_lexicon::Architecture::Powerpc64le),
platform_tags::Arch::X86 => Self(target_lexicon::Architecture::X86_32(
target_lexicon::X86_32Architecture::I686,
)),
platform_tags::Arch::X86_64 => Self(target_lexicon::Architecture::X86_64),
}
}
}
impl From<&platform_tags::Os> for Libc {
fn from(value: &platform_tags::Os) -> Self {
match value {
platform_tags::Os::Manylinux { .. } => Self::Some(target_lexicon::Environment::Gnu),
platform_tags::Os::Musllinux { .. } => Self::Some(target_lexicon::Environment::Musl),
_ => Self::None,
}
}
}
impl From<&platform_tags::Os> for Os {
fn from(value: &platform_tags::Os) -> Self {
match value {
platform_tags::Os::Dragonfly { .. } => Self(target_lexicon::OperatingSystem::Dragonfly),
platform_tags::Os::FreeBsd { .. } => Self(target_lexicon::OperatingSystem::Freebsd),
platform_tags::Os::Haiku { .. } => Self(target_lexicon::OperatingSystem::Haiku),
platform_tags::Os::Illumos { .. } => Self(target_lexicon::OperatingSystem::Illumos),
platform_tags::Os::Macos { .. } => Self(target_lexicon::OperatingSystem::Darwin),
platform_tags::Os::Manylinux { .. } | platform_tags::Os::Musllinux { .. } => {
Self(target_lexicon::OperatingSystem::Linux)
}
platform_tags::Os::NetBsd { .. } => Self(target_lexicon::OperatingSystem::Netbsd),
platform_tags::Os::OpenBsd { .. } => Self(target_lexicon::OperatingSystem::Openbsd),
platform_tags::Os::Windows => Self(target_lexicon::OperatingSystem::Windows),
}
}
}

View file

@ -1,3 +1,4 @@
use pep440_rs::Version;
use tracing::{debug, info};
use uv_client::BaseClientBuilder;
use uv_configuration::PreviewMode;
@ -9,7 +10,9 @@ use crate::discovery::{
ToolchainSources,
};
use crate::downloads::{DownloadResult, PythonDownload, PythonDownloadRequest};
use crate::implementation::LenientImplementationName;
use crate::managed::{InstalledToolchain, InstalledToolchains};
use crate::platform::{Arch, Libc, Os};
use crate::{Error, Interpreter, ToolchainSource};
/// A Python interpreter and accompanying tools.
@ -21,6 +24,15 @@ pub struct Toolchain {
}
impl Toolchain {
/// Create a new [`Toolchain`] from a source, interpreter tuple.
pub(crate) fn from_tuple(tuple: (ToolchainSource, Interpreter)) -> Self {
let (source, interpreter) = tuple;
Self {
source,
interpreter,
}
}
/// Find an installed [`Toolchain`].
///
/// This is the standard interface for discovering a Python toolchain for use with uv.
@ -144,6 +156,7 @@ impl Toolchain {
}
}
/// Download and install the requested toolchain.
pub async fn fetch<'a>(
request: ToolchainRequest,
client_builder: BaseClientBuilder<'a>,
@ -180,10 +193,48 @@ impl Toolchain {
}
}
/// Return the [`ToolchainSource`] of the toolchain, indicating where it was found.
pub fn source(&self) -> &ToolchainSource {
&self.source
}
pub fn key(&self) -> String {
format!(
"{}-{}-{}-{}-{}",
self.implementation().to_string().to_ascii_lowercase(),
self.python_version(),
self.os(),
self.arch(),
self.libc()
)
}
/// Return the Python [`Version`] of the toolchain as reported by its interpreter.
pub fn python_version(&self) -> &Version {
self.interpreter.python_version()
}
/// Return the [`LenientImplementationName`] of the toolchain as reported by its interpreter.
pub fn implementation(&self) -> LenientImplementationName {
LenientImplementationName::from(self.interpreter.implementation_name())
}
/// Return the [`Arch`] of the toolchain as reported by its interpreter.
pub fn arch(&self) -> Arch {
Arch::from(&self.interpreter.platform().arch())
}
/// Return the [`Libc`] of the toolchain as reported by its interpreter.
pub fn libc(&self) -> Libc {
Libc::from(self.interpreter.platform().os())
}
/// Return the [`Os`] of the toolchain as reported by its interpreter.
pub fn os(&self) -> Os {
Os::from(self.interpreter.platform().os())
}
/// Return the [`Interpreter`] for the toolchain.
pub fn interpreter(&self) -> &Interpreter {
&self.interpreter
}

View file

@ -2063,12 +2063,16 @@ pub(crate) enum ToolchainCommand {
#[derive(Args)]
#[allow(clippy::struct_excessive_bools)]
pub(crate) struct ToolchainListArgs {
/// List all available toolchains, including those that do not match the current platform.
#[arg(long, conflicts_with = "only_installed")]
pub(crate) all: bool,
/// List all toolchain versions, including outdated patch versions.
#[arg(long)]
pub(crate) all_versions: bool,
/// Only list installed toolchains.
#[arg(long, conflicts_with = "all")]
/// List toolchains for all platforms.
#[arg(long)]
pub(crate) all_platforms: bool,
/// Only show installed toolchains, exclude available downloads.
#[arg(long)]
pub(crate) only_installed: bool,
}

View file

@ -1,35 +1,50 @@
use std::collections::BTreeSet;
use std::collections::{BTreeSet, HashSet};
use std::fmt::Write;
use anyhow::Result;
use itertools::Itertools;
use uv_cache::Cache;
use uv_configuration::PreviewMode;
use uv_fs::Simplified;
use uv_toolchain::downloads::PythonDownloadRequest;
use uv_toolchain::managed::InstalledToolchains;
use uv_toolchain::{
find_toolchains, DiscoveryError, SystemPython, Toolchain, ToolchainNotFound, ToolchainRequest,
ToolchainSource, ToolchainSources,
};
use uv_warnings::warn_user;
use crate::commands::ExitStatus;
use crate::printer::Printer;
use crate::settings::ToolchainListIncludes;
use crate::settings::ToolchainListKinds;
#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)]
enum Kind {
Download,
Managed,
System,
}
/// List available toolchains.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list(
includes: ToolchainListIncludes,
kinds: ToolchainListKinds,
all_versions: bool,
all_platforms: bool,
preview: PreviewMode,
_cache: &Cache,
cache: &Cache,
printer: Printer,
) -> Result<ExitStatus> {
if preview.is_disabled() {
warn_user!("`uv toolchain list` is experimental and may change without warning.");
}
let download_request = match includes {
ToolchainListIncludes::All => Some(PythonDownloadRequest::default()),
ToolchainListIncludes::Installed => None,
ToolchainListIncludes::Default => Some(PythonDownloadRequest::from_env()?),
let download_request = match kinds {
ToolchainListKinds::Installed => None,
ToolchainListKinds::Default => Some(if all_platforms {
PythonDownloadRequest::default()
} else {
PythonDownloadRequest::from_env()?
}),
};
let downloads = download_request
@ -38,27 +53,68 @@ pub(crate) async fn list(
.into_iter()
.flatten();
let installed = {
InstalledToolchains::from_settings()?
.init()?
.find_all()?
.collect_vec()
};
let installed = find_toolchains(
&ToolchainRequest::Any,
SystemPython::Required,
&ToolchainSources::All(PreviewMode::Enabled),
cache,
)
// Raise any errors encountered during discovery
.collect::<Result<Vec<Result<Toolchain, ToolchainNotFound>>, DiscoveryError>>()?
.into_iter()
// Then drop any "missing" toolchains
.filter_map(std::result::Result::ok);
// Sort and de-duplicate the output.
let mut output = BTreeSet::new();
for toolchain in installed {
let kind = if matches!(toolchain.source(), ToolchainSource::Managed) {
Kind::Managed
} else {
Kind::System
};
output.insert((
toolchain.python_version().version().clone(),
toolchain.key().to_owned(),
toolchain.python_version().clone(),
toolchain.os().to_string(),
toolchain.key().clone(),
kind,
Some(toolchain.interpreter().sys_executable().to_path_buf()),
));
}
for download in downloads {
output.insert((download.python_version().version().clone(), download.key()));
output.insert((
download.python_version().version().clone(),
download.os().to_string(),
download.key().clone(),
Kind::Download,
None,
));
}
for (version, key) in output {
writeln!(printer.stdout(), "{:<8} ({key})", version.to_string())?;
let mut seen_minor = HashSet::new();
let mut seen_patch = HashSet::new();
for (version, os, key, kind, path) in output.iter().rev() {
// Only show the latest patch version for each download unless all were requested
if !matches!(kind, Kind::System) {
if let [major, minor, ..] = version.release() {
if !seen_minor.insert((os.clone(), *major, *minor)) {
if matches!(kind, Kind::Download) && !all_versions {
continue;
}
}
}
if let [major, minor, patch] = version.release() {
if !seen_patch.insert((os.clone(), *major, *minor, *patch)) {
if matches!(kind, Kind::Download) {
continue;
}
}
}
}
if let Some(path) = path {
writeln!(printer.stdout(), "{key}\t{}", path.user_display())?;
} else {
writeln!(printer.stdout(), "{key}\t<download available>")?;
}
}
Ok(ExitStatus::Success)

View file

@ -719,7 +719,15 @@ async fn run() -> Result<ExitStatus> {
// Initialize the cache.
let cache = cache.init()?;
commands::toolchain_list(args.includes, globals.preview, &cache, printer).await
commands::toolchain_list(
args.kinds,
args.all_versions,
args.all_platforms,
globals.preview,
&cache,
printer,
)
.await
}
Commands::Toolchain(ToolchainNamespace {
command: ToolchainCommand::Install(args),

View file

@ -205,10 +205,9 @@ impl ToolRunSettings {
}
#[derive(Debug, Clone, Default)]
pub(crate) enum ToolchainListIncludes {
pub(crate) enum ToolchainListKinds {
#[default]
Default,
All,
Installed,
}
@ -216,7 +215,9 @@ pub(crate) enum ToolchainListIncludes {
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone)]
pub(crate) struct ToolchainListSettings {
pub(crate) includes: ToolchainListIncludes,
pub(crate) kinds: ToolchainListKinds,
pub(crate) all_platforms: bool,
pub(crate) all_versions: bool,
}
impl ToolchainListSettings {
@ -224,19 +225,22 @@ impl ToolchainListSettings {
#[allow(clippy::needless_pass_by_value)]
pub(crate) fn resolve(args: ToolchainListArgs, _workspace: Option<Workspace>) -> Self {
let ToolchainListArgs {
all,
all_versions,
all_platforms,
only_installed,
} = args;
let includes = if all {
ToolchainListIncludes::All
} else if only_installed {
ToolchainListIncludes::Installed
let kinds = if only_installed {
ToolchainListKinds::Installed
} else {
ToolchainListIncludes::default()
ToolchainListKinds::default()
};
Self { includes }
Self {
kinds,
all_platforms,
all_versions,
}
}
}