From 47bbb7a78e3eb0505abb0e6171ee5db808a61aa1 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 5 Oct 2023 23:24:38 -0400 Subject: [PATCH] Separate platform tags (#18) --- crates/puffin-cli/src/commands/compile.rs | 16 +- crates/puffin-cli/src/commands/install.rs | 16 +- crates/puffin-package/Cargo.toml | 2 + crates/puffin-package/src/wheel.rs | 5 +- crates/puffin-platform/src/lib.rs | 264 +-------------------- crates/puffin-platform/src/tags.rs | 267 ++++++++++++++++++++++ crates/puffin-resolve/src/lib.rs | 11 +- 7 files changed, 300 insertions(+), 281 deletions(-) create mode 100644 crates/puffin-platform/src/tags.rs diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs index 80bf90214..aea032b50 100644 --- a/crates/puffin-cli/src/commands/compile.rs +++ b/crates/puffin-cli/src/commands/compile.rs @@ -5,6 +5,7 @@ use anyhow::Result; use tracing::debug; use puffin_interpreter::PythonExecutable; +use puffin_platform::tags::Tags; use puffin_platform::Platform; use puffin_resolve::resolve; @@ -25,15 +26,14 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result) -> Result bool { - for tag in compatible_tags { + pub fn is_compatible(&self, compatible_tags: &Tags) -> bool { + for tag in compatible_tags.iter() { if self.python_tag.contains(&tag.0) && self.abi_tag.contains(&tag.1) && self.platform_tag.contains(&tag.2) diff --git a/crates/puffin-platform/src/lib.rs b/crates/puffin-platform/src/lib.rs index fe48fcfe2..6ac23d7a0 100644 --- a/crates/puffin-platform/src/lib.rs +++ b/crates/puffin-platform/src/lib.rs @@ -11,6 +11,8 @@ use serde::Deserialize; use thiserror::Error; use tracing::trace; +pub mod tags; + #[derive(Error, Debug)] pub enum PlatformError { #[error(transparent)] @@ -35,13 +37,6 @@ impl Platform { pub fn is_windows(&self) -> bool { matches!(self.os, Os::Windows) } - - pub fn compatible_tags( - &self, - python_version: &pep440_rs::Version, - ) -> Result, PlatformError> { - compatible_tags(python_version, &self.os, self.arch) - } } /// All supported operating systems. @@ -233,6 +228,7 @@ impl Arch { } } +/// Get the macOS version from the SystemVersion.plist file. fn get_mac_os_version() -> Result<(u16, u16), PlatformError> { // This is actually what python does // https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428 @@ -266,37 +262,7 @@ fn get_mac_os_version() -> Result<(u16, u16), PlatformError> { } } -/// Determine the appropriate binary formats for a macOS version. -/// Source: -fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec { - let mut formats = vec![match arch { - Arch::Aarch64 => "arm64".to_string(), - _ => arch.to_string(), - }]; - - if matches!(arch, Arch::X86_64) { - if (major, minor) < (10, 4) { - return vec![]; - } - formats.extend([ - "intel".to_string(), - "fat64".to_string(), - "fat32".to_string(), - ]); - } - - if matches!(arch, Arch::X86_64 | Arch::Aarch64) { - formats.push("universal2".to_string()); - } - - if matches!(arch, Arch::X86_64) { - formats.push("universal".to_string()); - } - - formats -} - -/// Find musl libc path from executable's ELF header +/// Find musl libc path from executable's ELF header. fn find_libc() -> Result { let buffer = fs::read("/bin/ls")?; let error_str = "Couldn't parse /bin/ls for detecting the ld version"; @@ -311,13 +277,15 @@ fn find_libc() -> Result { } } -/// Read the musl version from libc library's output. Taken from maturin +/// Read the musl version from libc library's output. Taken from maturin. /// -/// The libc library should output something like this to `stderr::` +/// The libc library should output something like this to `stderr`: /// +/// ```text /// musl libc (`x86_64`) /// Version 1.2.2 /// Dynamic Program Loader +/// ``` fn get_musl_version(ld_path: impl AsRef) -> std::io::Result> { let output = Command::new(ld_path.as_ref()) .stdout(Stdio::null()) @@ -332,219 +300,3 @@ fn get_musl_version(ld_path: impl AsRef) -> std::io::Result Result, PlatformError> { - let platform_tags = match (&os, arch) { - (Os::Manylinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - platform_tags.extend( - (arch.get_minimum_manylinux_minor()..=*minor) - .map(|minor| format!("manylinux_{major}_{minor}_{arch}")), - ); - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) { - platform_tags.push(format!("manylinux2010_{arch}")); - } - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) { - platform_tags.push(format!("manylinux2014_{arch}")); - } - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) { - platform_tags.push(format!("manylinux1_{arch}")); - } - platform_tags - } - (Os::Musllinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - // musl 1.1 is the lowest supported version in musllinux - platform_tags - .extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}"))); - platform_tags - } - (Os::Macos { major, minor }, Arch::X86_64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - match major { - 10 => { - // Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version - // number. The major version was always 10. - for minor in (0..=*minor).rev() { - for binary_format in get_mac_binary_formats(*major, minor, arch) { - platform_tags.push(format!("macosx_{major}_{minor}_{binary_format}")); - } - } - } - value if *value >= 11 => { - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=*major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - for minor in (4..=16).rev() { - for binary_format in get_mac_binary_formats(10, minor, arch) { - platform_tags - .push(format!("macosx_{}_{}_{}", 10, minor, binary_format)); - } - } - } - _ => { - return Err(PlatformError::OsVersionDetectionError(format!( - "Unsupported macOS version: {major}", - ))); - } - } - platform_tags - } - (Os::Macos { major, .. }, Arch::Aarch64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=*major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - platform_tags.extend( - (4..=16) - .rev() - .map(|minor| format!("macosx_{}_{}_universal2", 10, minor)), - ); - platform_tags - } - (Os::Windows, Arch::X86) => { - vec!["win32".to_string()] - } - (Os::Windows, Arch::X86_64) => { - vec!["win_amd64".to_string()] - } - (Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()], - ( - Os::FreeBsd { release } - | Os::NetBsd { release } - | Os::OpenBsd { release } - | Os::Dragonfly { release } - | Os::Haiku { release }, - _, - ) => { - let release = release.replace(['.', '-'], "_"); - vec![format!( - "{}_{}_{}", - os.to_string().to_lowercase(), - release, - arch - )] - } - (Os::Illumos { release, arch }, _) => { - // See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730 - if let Some((major, other)) = release.split_once('_') { - let major_ver: u64 = major.parse().map_err(|err| { - PlatformError::OsVersionDetectionError(format!( - "illumos major version is not a number: {err}" - )) - })?; - if major_ver >= 5 { - // SunOS 5 == Solaris 2 - let os = "solaris".to_string(); - let release = format!("{}_{}", major_ver - 3, other); - let arch = format!("{arch}_64bit"); - return Ok(vec![format!("{}_{}_{}", os, release, arch)]); - } - } - - let os = os.to_string().to_lowercase(); - vec![format!("{}_{}_{}", os, release, arch)] - } - _ => { - return Err(PlatformError::OsVersionDetectionError(format!( - "Unsupported operating system and architecture combination: {os} {arch}" - ))); - } - }; - Ok(platform_tags) -} - -/// Returns the compatible tags in a (`python_tag`, `abi_tag`, `platform_tag`) format -pub fn compatible_tags( - python_version: &pep440_rs::Version, - os: &Os, - arch: Arch, -) -> Result, PlatformError> { - let python_version = (python_version.release[0], python_version.release[1]); - - let mut tags = Vec::new(); - let platform_tags = compatible_platform_tags(os, arch)?; - // 1. This exact c api version - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - format!( - "cp{}{}{}", - python_version.0, - python_version.1, - // hacky but that's legacy anyways - if python_version.1 <= 7 { "m" } else { "" } - ), - platform_tag.clone(), - )); - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - "none".to_string(), - platform_tag.clone(), - )); - } - // 2. abi3 and no abi (e.g. executable binary) - // For some reason 3.2 is the minimum python for the cp abi - for minor in 2..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, minor), - "abi3".to_string(), - platform_tag.clone(), - )); - } - } - // 3. no abi (e.g. executable binary) - for minor in 0..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - platform_tag.clone(), - )); - } - } - // 4. major only - for platform_tag in platform_tags { - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - platform_tag, - )); - } - // 5. no binary - for minor in 0..=python_version.1 { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - "any".to_string(), - )); - } - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - "any".to_string(), - )); - tags.sort(); - Ok(tags) -} diff --git a/crates/puffin-platform/src/tags.rs b/crates/puffin-platform/src/tags.rs new file mode 100644 index 000000000..84e49ab9f --- /dev/null +++ b/crates/puffin-platform/src/tags.rs @@ -0,0 +1,267 @@ +use crate::{Arch, Os, Platform, PlatformError}; + +/// A set of compatible tags for a given Python version and platform, in +/// (`python_tag`, `abi_tag`, `platform_tag`) format. +#[derive(Debug)] +pub struct Tags(Vec<(String, String, String)>); + +impl Tags { + /// Returns the compatible tags for the given Python version and platform. + pub fn from_env( + platform: &Platform, + python_version: &pep440_rs::Version, + ) -> Result { + let python_version = (python_version.release[0], python_version.release[1]); + + let platform_tags = platform.compatible_tags()?; + + let mut tags = Vec::with_capacity(5 * platform_tags.len()); + + // 1. This exact c api version + for platform_tag in &platform_tags { + tags.push(( + format!("cp{}{}", python_version.0, python_version.1), + format!( + "cp{}{}{}", + python_version.0, + python_version.1, + // hacky but that's legacy anyways + if python_version.1 <= 7 { "m" } else { "" } + ), + platform_tag.clone(), + )); + tags.push(( + format!("cp{}{}", python_version.0, python_version.1), + "none".to_string(), + platform_tag.clone(), + )); + } + // 2. abi3 and no abi (e.g. executable binary) + // For some reason 3.2 is the minimum python for the cp abi + for minor in 2..=python_version.1 { + for platform_tag in &platform_tags { + tags.push(( + format!("cp{}{}", python_version.0, minor), + "abi3".to_string(), + platform_tag.clone(), + )); + } + } + // 3. no abi (e.g. executable binary) + for minor in 0..=python_version.1 { + for platform_tag in &platform_tags { + tags.push(( + format!("py{}{}", python_version.0, minor), + "none".to_string(), + platform_tag.clone(), + )); + } + } + // 4. major only + for platform_tag in platform_tags { + tags.push(( + format!("py{}", python_version.0), + "none".to_string(), + platform_tag, + )); + } + // 5. no binary + for minor in 0..=python_version.1 { + tags.push(( + format!("py{}{}", python_version.0, minor), + "none".to_string(), + "any".to_string(), + )); + } + tags.push(( + format!("py{}", python_version.0), + "none".to_string(), + "any".to_string(), + )); + tags.sort(); + Ok(Self(tags)) + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +impl Platform { + /// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`, + /// `macosx_11_0_arm64`, or `win_amd64`). + /// + /// We have two cases: Actual platform specific tags (including "merged" tags such as universal2) + /// and "any". + /// + /// Bit of a mess, needs to be cleaned up. + fn compatible_tags(&self) -> Result, PlatformError> { + let os = &self.os; + let arch = self.arch; + + let platform_tags = match (&os, arch) { + (Os::Manylinux { major, minor }, _) => { + let mut platform_tags = vec![format!("linux_{}", arch)]; + platform_tags.extend( + (arch.get_minimum_manylinux_minor()..=*minor) + .map(|minor| format!("manylinux_{major}_{minor}_{arch}")), + ); + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) { + platform_tags.push(format!("manylinux2010_{arch}")); + } + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) { + platform_tags.push(format!("manylinux2014_{arch}")); + } + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) { + platform_tags.push(format!("manylinux1_{arch}")); + } + platform_tags + } + (Os::Musllinux { major, minor }, _) => { + let mut platform_tags = vec![format!("linux_{}", arch)]; + // musl 1.1 is the lowest supported version in musllinux + platform_tags + .extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}"))); + platform_tags + } + (Os::Macos { major, minor }, Arch::X86_64) => { + // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 + let mut platform_tags = vec![]; + match major { + 10 => { + // Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version + // number. The major version was always 10. + for minor in (0..=*minor).rev() { + for binary_format in get_mac_binary_formats(*major, minor, arch) { + platform_tags + .push(format!("macosx_{major}_{minor}_{binary_format}")); + } + } + } + value if *value >= 11 => { + // Starting with Mac OS 11, each yearly release bumps the major version number. + // The minor versions are now the midyear updates. + for major in (10..=*major).rev() { + for binary_format in get_mac_binary_formats(major, 0, arch) { + platform_tags + .push(format!("macosx_{}_{}_{}", major, 0, binary_format)); + } + } + // The "universal2" binary format can have a macOS version earlier than 11.0 + // when the x86_64 part of the binary supports that version of macOS. + for minor in (4..=16).rev() { + for binary_format in get_mac_binary_formats(10, minor, arch) { + platform_tags + .push(format!("macosx_{}_{}_{}", 10, minor, binary_format)); + } + } + } + _ => { + return Err(PlatformError::OsVersionDetectionError(format!( + "Unsupported macOS version: {major}", + ))); + } + } + platform_tags + } + (Os::Macos { major, .. }, Arch::Aarch64) => { + // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 + let mut platform_tags = vec![]; + // Starting with Mac OS 11, each yearly release bumps the major version number. + // The minor versions are now the midyear updates. + for major in (10..=*major).rev() { + for binary_format in get_mac_binary_formats(major, 0, arch) { + platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); + } + } + // The "universal2" binary format can have a macOS version earlier than 11.0 + // when the x86_64 part of the binary supports that version of macOS. + platform_tags.extend( + (4..=16) + .rev() + .map(|minor| format!("macosx_{}_{}_universal2", 10, minor)), + ); + platform_tags + } + (Os::Windows, Arch::X86) => { + vec!["win32".to_string()] + } + (Os::Windows, Arch::X86_64) => { + vec!["win_amd64".to_string()] + } + (Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()], + ( + Os::FreeBsd { release } + | Os::NetBsd { release } + | Os::OpenBsd { release } + | Os::Dragonfly { release } + | Os::Haiku { release }, + _, + ) => { + let release = release.replace(['.', '-'], "_"); + vec![format!( + "{}_{}_{}", + os.to_string().to_lowercase(), + release, + arch + )] + } + (Os::Illumos { release, arch }, _) => { + // See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730 + if let Some((major, other)) = release.split_once('_') { + let major_ver: u64 = major.parse().map_err(|err| { + PlatformError::OsVersionDetectionError(format!( + "illumos major version is not a number: {err}" + )) + })?; + if major_ver >= 5 { + // SunOS 5 == Solaris 2 + let os = "solaris".to_string(); + let release = format!("{}_{}", major_ver - 3, other); + let arch = format!("{arch}_64bit"); + return Ok(vec![format!("{}_{}_{}", os, release, arch)]); + } + } + + let os = os.to_string().to_lowercase(); + vec![format!("{}_{}_{}", os, release, arch)] + } + _ => { + return Err(PlatformError::OsVersionDetectionError(format!( + "Unsupported operating system and architecture combination: {os} {arch}" + ))); + } + }; + Ok(platform_tags) + } +} + +/// Determine the appropriate binary formats for a macOS version. +/// Source: +fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec { + let mut formats = vec![match arch { + Arch::Aarch64 => "arm64".to_string(), + _ => arch.to_string(), + }]; + + if matches!(arch, Arch::X86_64) { + if (major, minor) < (10, 4) { + return vec![]; + } + formats.extend([ + "intel".to_string(), + "fat64".to_string(), + "fat32".to_string(), + ]); + } + + if matches!(arch, Arch::X86_64 | Arch::Aarch64) { + formats.push("universal2".to_string()); + } + + if matches!(arch, Arch::X86_64) { + formats.push("universal".to_string()); + } + + formats +} diff --git a/crates/puffin-resolve/src/lib.rs b/crates/puffin-resolve/src/lib.rs index ed525c200..8216246b4 100644 --- a/crates/puffin-resolve/src/lib.rs +++ b/crates/puffin-resolve/src/lib.rs @@ -14,8 +14,9 @@ use puffin_package::metadata::Metadata21; use puffin_package::package_name::PackageName; use puffin_package::requirements::Requirements; use puffin_package::wheel::WheelFilename; -use puffin_platform::Platform; +use puffin_platform::tags::Tags; +#[derive(Debug)] pub struct Resolution(HashMap); impl Resolution { @@ -27,9 +28,8 @@ impl Resolution { /// Resolve a set of requirements into a set of pinned versions. pub async fn resolve( requirements: &Requirements, - python_version: &Version, markers: &MarkerEnvironment, - platform: &Platform, + tags: &Tags, cache: Option<&Path>, ) -> Result { // Instantiate a client. @@ -70,9 +70,6 @@ pub async fn resolve( in_flight.insert(PackageName::normalize(&requirement.name)); } - // Determine the compatible platform tags. - let tags = platform.compatible_tags(python_version)?; - // Resolve the requirements. let mut resolution: HashMap = HashMap::with_capacity(requirements.len()); @@ -102,7 +99,7 @@ pub async fn resolve( return false; }; - if !name.is_compatible(&tags) { + if !name.is_compatible(tags) { return false; }