Refactor uv-toolchain::platform to use target-lexicon (#4236)

Closes https://github.com/astral-sh/uv/issues/3857

Instead of using custom `Arch`, `Os`, and `Libc` types I just use
`target-lexicon`'s which enumerate way more variants and implement
display and parsing. We use a wrapper type to represent a couple special
cases to support the "x86" alias for "i686" and "macos" for "darwin".
Alternatively we could try to use our `platform-tags` types but those
capture more information (like operating system versions) that we don't
have for downloads.

As discussed in https://github.com/astral-sh/uv/pull/4160, this is not
sufficient for proper libc detection but that work is larger and will be
handled separately.

Cargo.lock

@@ -4921,6 +4921,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "target-lexicon",
  "temp-env",
  "tempfile",
  "test-log",

Cargo.toml

@@ -120,6 +120,7 @@ serde = { version = "1.0.197", features = ["derive"] }
 serde_json = { version = "1.0.114" }
 sha2 = { version = "0.10.8" }
 sys-info = { version = "0.9.1" }
+target-lexicon = {version = "0.12.14" }
 tempfile = { version = "3.9.0" }
 textwrap = { version = "0.16.1" }
 thiserror = { version = "1.0.56" }

crates/uv-toolchain/Cargo.toml

@@ -41,6 +41,7 @@ same-file = { workspace = true }
 schemars = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+target-lexicon = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio-util = { workspace = true, features = ["compat"] }

crates/uv-toolchain/fetch-download-metadata.py

@@ -84,11 +84,7 @@ _suffix_re = re.compile(
 ARCH_MAP = {
     "ppc64": "powerpc64",
     "ppc64le": "powerpc64le",
-    "i686": "x86",
-    "i386": "x86",
-    "armv7": "armv7l",
 }
-OS_MAP = {"darwin": "macos"}
 
 
 def parse_filename(filename):
@@ -107,8 +103,8 @@ def parse_filename(filename):
 
 
 def normalize_triple(triple):
-    if "-static" in triple or "-gnueabihf" in triple or "-gnueabi" in triple:
+    if "-static" in triple:
-        logging.debug("Skipping %r: unknown triple", triple)
+        logging.debug("Skipping %r: static unsupported", triple)
         return
     triple = SPECIAL_TRIPLES.get(triple, triple)
     pieces = triple.split("-")
@@ -134,7 +130,7 @@ def normalize_arch(arch):
 
 
 def normalize_os(os):
-    return OS_MAP.get(os, os)
+    return os
 
 
 def read_sha256(url):

crates/uv-toolchain/src/downloads.inc.mustache

@@ -6,14 +6,18 @@
 pub(crate) const PYTHON_DOWNLOADS: &[PythonDownload] = &[
     {{#versions}}
     PythonDownload {
-        key: "{{key}}",
         major: {{value.major}},
         minor: {{value.minor}},
         patch: {{value.patch}},
         implementation: ImplementationName::{{value.name}},
-        arch: Arch::{{value.arch}},
+        arch: Arch(target_lexicon::Architecture::{{value.arch}}),
-        os: Os::{{value.os}},
+        os: Os(target_lexicon::OperatingSystem::{{value.os}}),
-        libc: Libc::{{value.libc}},
+        {{#value.libc}}
+        libc: Libc::Some(target_lexicon::Environment::{{.}}),
+        {{/value.libc}}
+        {{^value.libc}}
+        libc: Libc::None,
+        {{/value.libc}}
         url: "{{value.url}}",
         {{#value.sha256}}
         sha256: Some("{{.}}")

crates/uv-toolchain/src/downloads.rs

@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
 use crate::implementation::{Error as ImplementationError, ImplementationName};
-use crate::platform::{Arch, Error as PlatformError, Libc, Os};
+use crate::platform::{Arch, Libc, Os};
 use crate::{PythonVersion, ToolchainRequest, VersionRequest};
 use thiserror::Error;
 use uv_client::BetterReqwestError;
@@ -21,8 +21,6 @@ pub enum Error {
     #[error(transparent)]
     IO(#[from] io::Error),
     #[error(transparent)]
-    PlatformError(#[from] PlatformError),
-    #[error(transparent)]
     ImplementationError(#[from] ImplementationError),
     #[error("Invalid python version: {0}")]
     InvalidPythonVersion(String),
@@ -59,7 +57,6 @@ pub enum Error {
 
 #[derive(Debug, PartialEq)]
 pub struct PythonDownload {
-    key: &'static str,
     implementation: ImplementationName,
     arch: Arch,
     os: Os,
@@ -157,10 +154,10 @@ impl PythonDownloadRequest {
             self.implementation = Some(ImplementationName::CPython);
         }
         if self.arch.is_none() {
-            self.arch = Some(Arch::from_env()?);
+            self.arch = Some(Arch::from_env());
         }
         if self.os.is_none() {
-            self.os = Some(Os::from_env()?);
+            self.os = Some(Os::from_env());
         }
         if self.libc.is_none() {
             self.libc = Some(Libc::from_env());
@@ -173,8 +170,8 @@ impl PythonDownloadRequest {
         Ok(Self::new(
             None,
             None,
-            Some(Arch::from_env()?),
+            Some(Arch::from_env()),
-            Some(Os::from_env()?),
+            Some(Os::from_env()),
             Some(Libc::from_env()),
         ))
     }
@@ -252,11 +249,6 @@ pub enum DownloadResult {
 }
 
 impl PythonDownload {
-    /// Return the [`PythonDownload`] corresponding to the key, if it exists.
-    pub fn from_key(key: &str) -> Option<&PythonDownload> {
-        PYTHON_DOWNLOADS.iter().find(|&value| value.key == key)
-    }
-
     /// Return the first [`PythonDownload`] matching a request, if any.
     pub fn from_request(request: &PythonDownloadRequest) -> Result<&'static PythonDownload, Error> {
         request
@@ -274,8 +266,17 @@ impl PythonDownload {
         self.url
     }
 
-    pub fn key(&self) -> &str {
+    pub fn key(&self) -> String {
-        self.key
+        format!(
+            "{}-{}.{}.{}-{}-{}-{}",
+            self.implementation.as_str().to_ascii_lowercase(),
+            self.major,
+            self.minor,
+            self.patch,
+            self.os,
+            self.arch,
+            self.libc
+        )
     }
 
     pub fn sha256(&self) -> Option<&str> {
@@ -289,7 +290,7 @@ impl PythonDownload {
         parent_path: &Path,
     ) -> Result<DownloadResult, Error> {
         let url = Url::parse(self.url)?;
-        let path = parent_path.join(self.key).clone();
+        let path = parent_path.join(self.key()).clone();
 
         // If it already exists, return it
         if path.is_dir() {
@@ -363,6 +364,6 @@ impl From<reqwest_middleware::Error> for Error {
 
 impl Display for PythonDownload {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(self.key)
+        f.write_str(&self.key())
     }
 }

crates/uv-toolchain/src/managed.rs

@@ -155,7 +155,7 @@ impl InstalledToolchains {
     pub fn find_matching_current_platform(
         &self,
     ) -> Result<impl DoubleEndedIterator<Item = InstalledToolchain>, Error> {
-        let platform_key = platform_key_from_env()?;
+        let platform_key = platform_key_from_env();
 
         let iter = InstalledToolchains::from_settings()?
             .find_all()?
@@ -286,11 +286,11 @@ impl InstalledToolchain {
 }
 
 /// Generate a platform portion of a key from the environment.
-fn platform_key_from_env() -> Result<String, Error> {
+fn platform_key_from_env() -> String {
-    let os = Os::from_env()?;
+    let os = Os::from_env();
-    let arch = Arch::from_env()?;
+    let arch = Arch::from_env();
     let libc = Libc::from_env();
-    Ok(format!("{os}-{arch}-{libc}").to_lowercase())
+    format!("{os}-{arch}-{libc}").to_lowercase()
 }
 
 impl fmt::Display for InstalledToolchain {

crates/uv-toolchain/src/platform.rs

@@ -1,72 +1,80 @@
-use std::{
+use std::fmt::Display;
-    fmt::{self},
+use std::ops::Deref;
-    str::FromStr,
+use std::{fmt, str::FromStr};
-};
 use thiserror::Error;
 
-/// All supported operating systems.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum Os {
-    Windows,
-    Linux,
-    Macos,
-    FreeBsd,
-    NetBsd,
-    OpenBsd,
-    Dragonfly,
-    Illumos,
-    Haiku,
-}
-
-/// All supported CPU architectures
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
-pub enum Arch {
-    Aarch64,
-    Armv6L,
-    Armv7L,
-    Powerpc64Le,
-    Powerpc64,
-    X86,
-    X86_64,
-    S390X,
-}
-
-#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-pub enum Libc {
-    Gnu,
-    Musl,
-    None,
-}
-
 #[derive(Error, Debug)]
 pub enum Error {
-    #[error("Operating system not supported: {0}")]
+    #[error("Unknown operating system: {0}")]
-    OsNotSupported(String),
+    UnknownOs(String),
-    #[error("Architecture not supported: {0}")]
+    #[error("Unknown architecture: {0}")]
-    ArchNotSupported(String),
+    UnknownArch(String),
-    #[error("Libc type could not be detected")]
-    LibcNotDetected,
 }
 
-impl fmt::Display for Os {
+#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+pub struct Arch(pub(crate) target_lexicon::Architecture);
-        match *self {
+
-            Self::Windows => write!(f, "Windows"),
+#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-            Self::Macos => write!(f, "MacOS"),
+pub struct Os(pub(crate) target_lexicon::OperatingSystem);
-            Self::FreeBsd => write!(f, "FreeBSD"),
+
-            Self::NetBsd => write!(f, "NetBSD"),
+#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-            Self::Linux => write!(f, "Linux"),
+pub enum Libc {
-            Self::OpenBsd => write!(f, "OpenBSD"),
+    Some(target_lexicon::Environment),
-            Self::Dragonfly => write!(f, "DragonFly"),
+    None,
-            Self::Illumos => write!(f, "Illumos"),
+}
-            Self::Haiku => write!(f, "Haiku"),
+
+impl Libc {
+    pub(crate) fn from_env() -> Self {
+        match std::env::consts::OS {
+            // TODO(zanieb): On Linux, we use the uv target host to determine the libc variant
+            // but we should only use this as a fallback and should instead inspect the
+            // machine's `/bin/sh` (or similar).
+            "linux" => Self::Some(target_lexicon::Environment::Gnu),
+            "windows" | "macos" => Self::None,
+            // Use `None` on platforms without explicit support.
+            _ => Self::None,
         }
     }
 }
 
 impl Os {
-    pub(crate) fn from_env() -> Result<Self, Error> {
+    pub(crate) fn from_env() -> Self {
-        Self::from_str(std::env::consts::OS)
+        Self(target_lexicon::HOST.operating_system)
+    }
+}
+
+impl Arch {
+    pub(crate) fn from_env() -> Self {
+        Self(target_lexicon::HOST.architecture)
+    }
+}
+
+impl Display for Libc {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Some(env) => write!(f, "{env}"),
+            Self::None => write!(f, "none"),
+        }
+    }
+}
+
+impl Display for Os {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &**self {
+            target_lexicon::OperatingSystem::Darwin => write!(f, "macos"),
+            inner => write!(f, "{inner}"),
+        }
+    }
+}
+
+impl Display for Arch {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &**self {
+            target_lexicon::Architecture::X86_32(target_lexicon::X86_32Architecture::I686) => {
+                write!(f, "x86")
+            }
+            inner => write!(f, "{inner}"),
+        }
     }
 }
 
@@ -74,33 +82,15 @@ impl FromStr for Os {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s.to_lowercase().as_str() {
+        let inner = match s {
-            "windows" => Ok(Self::Windows),
+            "macos" => target_lexicon::OperatingSystem::Darwin,
-            "linux" => Ok(Self::Linux),
+            _ => target_lexicon::OperatingSystem::from_str(s)
-            "macos" => Ok(Self::Macos),
+                .map_err(|()| Error::UnknownOs(s.to_string()))?,
-            "freebsd" => Ok(Self::FreeBsd),
+        };
-            "netbsd" => Ok(Self::NetBsd),
+        if matches!(inner, target_lexicon::OperatingSystem::Unknown) {
-            "openbsd" => Ok(Self::OpenBsd),
+            return Err(Error::UnknownOs(s.to_string()));
-            "dragonfly" => Ok(Self::Dragonfly),
-            "illumos" => Ok(Self::Illumos),
-            "haiku" => Ok(Self::Haiku),
-            _ => Err(Error::OsNotSupported(s.to_string())),
-        }
-    }
-}
-
-impl fmt::Display for Arch {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
-            Self::Aarch64 => write!(f, "aarch64"),
-            Self::Armv6L => write!(f, "armv6l"),
-            Self::Armv7L => write!(f, "armv7l"),
-            Self::Powerpc64Le => write!(f, "ppc64le"),
-            Self::Powerpc64 => write!(f, "ppc64"),
-            Self::X86 => write!(f, "i686"),
-            Self::X86_64 => write!(f, "x86_64"),
-            Self::S390X => write!(f, "s390x"),
         }
+        Ok(Self(inner))
     }
 }
 
@@ -108,45 +98,32 @@ impl FromStr for Arch {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s.to_lowercase().as_str() {
+        let inner = match s {
-            "aarch64" | "arm64" => Ok(Self::Aarch64),
+            // Allow users to specify "x86" as a shorthand for the "i686" variant, they should not need
-            "armv6l" => Ok(Self::Armv6L),
+            // to specify the exact architecture and this variant is what we have downloads for.
-            "armv7l" | "arm" => Ok(Self::Armv7L),
+            "x86" => target_lexicon::Architecture::X86_32(target_lexicon::X86_32Architecture::I686),
-            "powerpc64le" | "ppc64le" => Ok(Self::Powerpc64Le),
+            _ => target_lexicon::Architecture::from_str(s)
-            "powerpc64" | "ppc64" => Ok(Self::Powerpc64),
+                .map_err(|()| Error::UnknownArch(s.to_string()))?,
-            "x86" | "i686" | "i386" => Ok(Self::X86),
+        };
-            "x86_64" | "amd64" => Ok(Self::X86_64),
+        if matches!(inner, target_lexicon::Architecture::Unknown) {
-            "s390x" => Ok(Self::S390X),
+            return Err(Error::UnknownArch(s.to_string()));
-            _ => Err(Error::ArchNotSupported(s.to_string())),
         }
+        Ok(Self(inner))
     }
 }
 
-impl Arch {
+impl Deref for Arch {
-    pub(crate) fn from_env() -> Result<Self, Error> {
+    type Target = target_lexicon::Architecture;
-        Self::from_str(std::env::consts::ARCH)
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
     }
 }
 
-impl Libc {
+impl Deref for Os {
-    pub(crate) fn from_env() -> Self {
+    type Target = target_lexicon::OperatingSystem;
-        // TODO(zanieb): Perform this lookup
-        match std::env::consts::OS {
-            // Supported platforms.
-            "linux" => Libc::Gnu,
-            "windows" | "macos" => Libc::None,
-            // Platforms without explicit support.
-            _ => Libc::None,
-        }
-    }
-}
 
-impl fmt::Display for Libc {
+    fn deref(&self) -> &Self::Target {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        &self.0
-        match self {
-            Libc::Gnu => f.write_str("gnu"),
-            Libc::None => f.write_str("none"),
-            Libc::Musl => f.write_str("musl"),
-        }
     }
 }

crates/uv-toolchain/template-download-metadata.py

@@ -41,11 +41,31 @@ def prepare_name(name: str) -> str:
             raise ValueError(f"Unknown implementation name: {name}")
 
 
+def prepare_libc(libc: str) -> str | None:
+    if libc == "none":
+        return None
+    else:
+        return libc.title()
+
+
+def prepare_arch(arch: str) -> str:
+    match arch:
+        # Special constructors
+        case "i686":
+            return "X86_32(target_lexicon::X86_32Architecture::I686)"
+        case "aarch64":
+            return "Aarch64(target_lexicon::Aarch64Architecture::Aarch64)"
+        case "armv7":
+            return "Arm(target_lexicon::ArmArchitecture::Armv7)"
+        case _:
+            return arch.capitalize()
+
+
 def prepare_value(value: dict) -> dict:
-    # Convert fields from snake case to camel case for enums
+    value["os"] = value["os"].title()
-    for key in ["arch", "os", "libc"]:
+    value["arch"] = prepare_arch(value["arch"])
-        value[key] = value[key].title()
     value["name"] = prepare_name(value["name"])
+    value["libc"] = prepare_libc(value["libc"])
     return value
 
 

crates/uv/src/commands/toolchain/list.rs

@@ -54,10 +54,7 @@ pub(crate) async fn list(
         ));
     }
     for download in downloads {
-        output.insert((
+        output.insert((download.python_version().version().clone(), download.key()));
-            download.python_version().version().clone(),
-            download.key().to_owned(),
-        ));
     }
 
     for (version, key) in output {