From ae28552b3a83b5847303ce49902e9783c58d72ca Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 6 Oct 2023 21:43:55 -0400 Subject: [PATCH] Use local copy of `install-wheel-rs` (#34) This PR modifies the `install-wheel-rs` (and a few other crates) to get everything playing nicely. Specifically, CI should pass, and all these crates now use workspace dependencies between one another. As part of this change, I split out the wheel name parsing into its own `wheel-filename` crate, and the compatibility tag parsing into its own `platform-tags` crate. --- Cargo.lock | 71 +- Cargo.toml | 9 +- crates/README.md | 16 +- crates/install-wheel-rs/Cargo.toml | 38 +- crates/install-wheel-rs/install_wheel_rs.pyi | 3 - crates/install-wheel-rs/pyproject.toml | 9 - crates/install-wheel-rs/run_tests.sh | 12 - .../install-wheel-rs/src/install_location.rs | 4 +- crates/install-wheel-rs/src/lib.rs | 68 +- crates/install-wheel-rs/src/main.rs | 7 +- .../install-wheel-rs/src/python_bindings.rs | 19 +- crates/install-wheel-rs/src/wheel.rs | 220 ++-- crates/install-wheel-rs/src/wheel_tags.rs | 937 ------------------ .../upsidedown-0.4-py2.py3-none-any.whl | Bin 5495 -> 0 bytes .../test/test_install_wheel_rs.py | 35 - .../Cargo.toml | 5 +- .../src/lib.rs | 10 +- crates/platform-tags/Cargo.toml | 13 + crates/platform-tags/src/lib.rs | 258 +++++ crates/puffin-cli/Cargo.toml | 8 +- crates/puffin-cli/src/commands/compile.rs | 6 +- crates/puffin-cli/src/commands/sync.rs | 6 +- crates/puffin-installer/Cargo.toml | 3 +- crates/puffin-installer/src/lib.rs | 8 +- crates/puffin-interpreter/Cargo.toml | 2 +- crates/puffin-interpreter/src/lib.rs | 4 +- crates/puffin-interpreter/src/markers.rs | 1 + .../puffin-interpreter/src/python_platform.rs | 6 +- crates/puffin-package/Cargo.toml | 2 +- crates/puffin-package/src/lib.rs | 1 - crates/puffin-package/src/metadata.rs | 5 +- crates/puffin-package/src/requirements.rs | 1 + crates/puffin-platform/src/tags.rs | 267 ----- crates/puffin-resolver/Cargo.toml | 4 +- crates/puffin-resolver/src/lib.rs | 8 +- crates/wheel-filename/Cargo.toml | 15 + .../wheel.rs => wheel-filename/src/lib.rs} | 2 +- 37 files changed, 516 insertions(+), 1567 deletions(-) delete mode 100644 crates/install-wheel-rs/install_wheel_rs.pyi delete mode 100644 crates/install-wheel-rs/pyproject.toml delete mode 100644 crates/install-wheel-rs/run_tests.sh delete mode 100644 crates/install-wheel-rs/src/wheel_tags.rs delete mode 100644 crates/install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl delete mode 100644 crates/install-wheel-rs/test/test_install_wheel_rs.py rename crates/{puffin-platform => platform-host}/Cargo.toml (95%) rename crates/{puffin-platform => platform-host}/src/lib.rs (99%) create mode 100644 crates/platform-tags/Cargo.toml create mode 100644 crates/platform-tags/src/lib.rs delete mode 100644 crates/puffin-platform/src/tags.rs create mode 100644 crates/wheel-filename/Cargo.toml rename crates/{puffin-package/src/wheel.rs => wheel-filename/src/lib.rs} (98%) diff --git a/Cargo.lock b/Cargo.lock index 9574b7ead..48c5c17c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1111,20 +1111,23 @@ dependencies = [ [[package]] name = "install-wheel-rs" version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd8c1fd75d1c686f0e18aa3dd60532f119578b32501517576dd539e0be65f1e" dependencies = [ - "base64 0.21.4", + "clap", "configparser", "csv", + "data-encoding", "fs-err", "fs2", "glibc_version", "goblin", + "indoc 2.0.4", "mailparse", "once_cell", + "platform-host", "platform-info", "plist", + "pyo3", + "rayon", "regex", "rfc2047-decoder", "serde", @@ -1134,7 +1137,9 @@ dependencies = [ "tempfile", "thiserror", "tracing", + "tracing-subscriber", "walkdir", + "wheel-filename", "zip", ] @@ -1561,6 +1566,22 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "platform-host" +version = "0.1.0" +dependencies = [ + "glibc_version", + "goblin", + "pep440_rs", + "platform-info", + "plist", + "regex", + "serde", + "target-lexicon", + "thiserror", + "tracing", +] + [[package]] name = "platform-info" version = "2.0.2" @@ -1571,6 +1592,13 @@ dependencies = [ "winapi", ] +[[package]] +name = "platform-tags" +version = "0.1.0" +dependencies = [ + "platform-host", +] + [[package]] name = "plist" version = "1.5.0" @@ -1646,14 +1674,14 @@ dependencies = [ "colored", "directories", "futures", - "install-wheel-rs", "pep440_rs", "pep508_rs", + "platform-host", + "platform-tags", "puffin-client", "puffin-installer", "puffin-interpreter", "puffin-package", - "puffin-platform", "puffin-resolver", "tempfile", "tokio", @@ -1693,6 +1721,7 @@ dependencies = [ "tokio-util", "tracing", "url", + "wheel-filename", ] [[package]] @@ -1702,7 +1731,7 @@ dependencies = [ "anyhow", "pep440_rs", "pep508_rs", - "puffin-platform", + "platform-host", "serde_json", "tracing", ] @@ -1719,29 +1748,13 @@ dependencies = [ "once_cell", "pep440_rs", "pep508_rs", - "puffin-platform", + "platform-host", "regex", "rfc2047-decoder", "serde", "thiserror", ] -[[package]] -name = "puffin-platform" -version = "0.1.0" -dependencies = [ - "glibc_version", - "goblin", - "pep440_rs", - "platform-info", - "plist", - "regex", - "serde", - "target-lexicon", - "thiserror", - "tracing", -] - [[package]] name = "puffin-resolver" version = "0.1.0" @@ -1751,10 +1764,12 @@ dependencies = [ "futures", "pep440_rs", "pep508_rs", + "platform-host", + "platform-tags", "puffin-client", "puffin-package", - "puffin-platform", "tracing", + "wheel-filename", ] [[package]] @@ -2878,6 +2893,14 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wheel-filename" +version = "0.1.0" +dependencies = [ + "platform-tags", + "thiserror", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 881f247df..fc9c734a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,16 @@ anyhow = { version = "1.0.75" } bitflags = { version = "2.4.0" } clap = { version = "4.4.6" } colored = { version = "2.0.4" } +configparser = { version = "3.0.2" } +csv = { version = "1.3.0" } +data-encoding = { version = "2.4.0" } directories = { version = "5.0.1" } +fs-err = { version = "2.9.0" } +fs2 = { version = "0.4.3" } futures = { version = "0.3.28" } glibc_version = { version = "0.1.2" } goblin = { version = "0.7.1" } http-cache-reqwest = { version = "0.11.3" } -install-wheel-rs = { version = "0.0.1" } mailparse = { version = "0.14.0" } memchr = { version = "2.6.4" } once_cell = { version = "1.18.0" } @@ -34,6 +38,7 @@ reqwest-retry = { version = "0.3.0" } rfc2047-decoder = { version = "1.0.1" } serde = { version = "1.0.188" } serde_json = { version = "1.0.107" } +sha2 = { version = "0.10.8" } target-lexicon = { version = "0.12.11" } tempfile = { version = "3.8.0" } thiserror = { version = "1.0.49" } @@ -44,3 +49,5 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } tracing-tree = { version = "0.2.5" } unicode-width = { version = "0.1.8" } url = { version = "2.4.1" } +walkdir = { version = "2.4.0" } +zip = { version = "0.6.6", default-features = false, features = ["deflate"] } diff --git a/crates/README.md b/crates/README.md index ca398296c..96dcfe776 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,6 +8,14 @@ Utilities for interacting with Python version numbers and specifiers. Utilities for interacting with [PEP 508](https://peps.python.org/pep-0508/) dependency specifiers. +## [platform-host](./platform-host) + +Functionality for detecting the current platform (operating system, architecture, etc.). + +## [platform-tags](./platform-tags) + +Functionality for parsing and inferring Python platform tags as per [PEP 425](https://peps.python.org/pep-0425/). + ## [puffin-cli](./puffin-cli) Command-line interface for the Puffin package manager. @@ -28,10 +36,10 @@ Functionality for detecting and leveraging the current Python interpreter. Types and functionality for working with Python packages, e.g., parsing wheel files. -## [puffin-platform](./puffin-platform) - -Functionality for detecting the current platform (operating system, architecture, etc.). - ## [puffin-resolver](./puffin-resolver) Functionality for resolving Python packages and their dependencies. + +## [wheel-filename](./wheel-filename) + +Functionality for parsing wheel filenames as per [PEP 427](https://peps.python.org/pep-0427/). diff --git a/crates/install-wheel-rs/Cargo.toml b/crates/install-wheel-rs/Cargo.toml index 69c00cdf7..506c34a01 100644 --- a/crates/install-wheel-rs/Cargo.toml +++ b/crates/install-wheel-rs/Cargo.toml @@ -9,36 +9,37 @@ keywords = ["wheel", "python"] [lib] name = "install_wheel_rs" -# https://github.com/PyO3/maturin/issues/1080 :(( -#crate-type = ["cdylib", "rlib"] [dependencies] -clap = { version = "4.4.6", optional = true, features = ["derive", "env"] } -configparser = "3.0.2" -csv = "1.2.2" -data-encoding = "2.4.0" +platform-host = { path = "../platform-host" } +wheel-filename = { path = "../wheel-filename" } + +clap = { workspace = true, optional = true, features = ["derive", "env"] } +configparser = { workspace = true } +csv = { workspace = true } +data-encoding = { workspace = true } fs-err = { workspace = true } fs2 = { workspace = true } -glibc_version = "0.1.2" -goblin = "0.7.1" -mailparse = "0.14.0" -once_cell = "1.18.0" -platform-info = "2.0.2" -plist = "1.5.0" -pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true } +glibc_version = { workspace = true } +goblin = { workspace = true } +mailparse = { workspace = true } +once_cell = { workspace = true } +platform-info = { workspace = true } +plist = { workspace = true } +pyo3 = { version = "0.19.2", features = ["extension-module", "abi3-py37"], optional = true } rayon = { version = "1.8.0", optional = true } regex = { workspace = true } -rfc2047-decoder = "1.0.1" +rfc2047-decoder = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha2 = { workspace = true } -target-lexicon = "0.12.11" +target-lexicon = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, optional = true } walkdir = { workspace = true } -zip = { version = "0.6.6", default-features = false, features = ["deflate"] } # no default features for zstd +zip = { workspace = true } [features] default = ["cli", "parallel"] @@ -47,7 +48,4 @@ cli = ["clap"] parallel = ["rayon"] [dev-dependencies] -indoc = { workspace = true } - -[package.metadata.dist] -dist = false +indoc = {version = "2.0.4"} diff --git a/crates/install-wheel-rs/install_wheel_rs.pyi b/crates/install-wheel-rs/install_wheel_rs.pyi deleted file mode 100644 index c2214beef..000000000 --- a/crates/install-wheel-rs/install_wheel_rs.pyi +++ /dev/null @@ -1,3 +0,0 @@ -class LockedVenv: - def __init__(self, venv: str): ... - def install_wheel(self, wheel: str): ... diff --git a/crates/install-wheel-rs/pyproject.toml b/crates/install-wheel-rs/pyproject.toml deleted file mode 100644 index b81f0f237..000000000 --- a/crates/install-wheel-rs/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[project] -name = "install-wheel-rs" - -[tool.maturin] -features = ["python_bindings"] - -[build-system] -requires = ["maturin>=1.2,<2.0"] -build-backend = "maturin" diff --git a/crates/install-wheel-rs/run_tests.sh b/crates/install-wheel-rs/run_tests.sh deleted file mode 100644 index 7b57c4fe8..000000000 --- a/crates/install-wheel-rs/run_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -# cd to project root -cd "$(git rev-parse --show-toplevel)" -rm -f target-maturin/wheels/install_wheel_rs-*.whl -CARGO_TARGET_DIR=target-maturin maturin build --release --strip --no-sdist -m install-wheel-rs/Cargo.toml -.venv/bin/pip uninstall -y install-wheel-rs -.venv/bin/pip install target-maturin/wheels/install_wheel_rs-*.whl -.venv/bin/pytest install-wheel-rs/test -cargo test \ No newline at end of file diff --git a/crates/install-wheel-rs/src/install_location.rs b/crates/install-wheel-rs/src/install_location.rs index 29fade9bd..9da800f9e 100644 --- a/crates/install-wheel-rs/src/install_location.rs +++ b/crates/install-wheel-rs/src/install_location.rs @@ -139,11 +139,11 @@ impl> InstallLocation { .join("site-packages") }; site_packages - .join(format!("{}-{}.dist-info", normalized_name, version)) + .join(format!("{normalized_name}-{version}.dist-info")) .is_dir() } InstallLocation::Monotrail { monotrail_root, .. } => monotrail_root - .join(format!("{}-{}", normalized_name, version)) + .join(format!("{normalized_name}-{version}")) .is_dir(), } } diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index c73020d12..c2f31dceb 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -1,37 +1,22 @@ -//! Takes a wheel and installs it, either in a venv or for monotrail -//! -//! ```no_run -//! use std::path::Path; -//! use install_wheel_rs::install_wheel_in_venv; -//! -//! install_wheel_in_venv( -//! "Django-4.2.6-py3-none-any.whl", -//! ".venv", -//! ".venv/bin/python", -//! (3, 8), -//! ).unwrap(); -//! ``` +//! Takes a wheel and installs it, either in a venv or for monotrail. + +use std::io; use platform_info::PlatformInfoError; -use std::fs::File; -use std::io; -use std::path::Path; -use std::str::FromStr; use thiserror::Error; use zip::result::ZipError; pub use install_location::{normalize_name, InstallLocation, LockedDir}; +use platform_host::{Arch, Os}; pub use wheel::{ get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to, Script, SHEBANG_PYTHON, }; -pub use wheel_tags::{Arch, CompatibleTags, Os, WheelFilename}; mod install_location; #[cfg(feature = "python_bindings")] mod python_bindings; mod wheel; -mod wheel_tags; #[derive(Error, Debug)] pub enum Error { @@ -50,8 +35,8 @@ pub enum Error { #[error("The poetry dependency specification (pyproject.toml or poetry.lock) is broken (try `poetry update`?): {0}")] InvalidPoetry(String), /// Doesn't follow file name schema - #[error("The wheel filename \"{0}\" is invalid: {1}")] - InvalidWheelFileName(String, String), + #[error(transparent)] + InvalidWheelFileName(#[from] wheel_filename::Error), #[error("Failed to read the wheel file {0}")] Zip(String, #[source] ZipError), #[error("Failed to run python subcommand")] @@ -80,44 +65,3 @@ impl Error { } } } - -/// High level API: Install a wheel in a virtualenv -/// -/// The python interpreter is used for compiling to byte code, the python version for computing -/// the site packages path on unix. -/// -/// Returns the tag of the wheel -pub fn install_wheel_in_venv( - wheel: impl AsRef, - venv: impl AsRef, - interpreter: impl AsRef, - major_minor: (u8, u8), -) -> Result { - let venv_base = venv.as_ref().canonicalize()?; - let location = InstallLocation::Venv { - venv_base, - python_version: major_minor, - }; - let locked_dir = location.acquire_lock()?; - - let filename = wheel - .as_ref() - .file_name() - .ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))? - .to_string_lossy(); - let filename = WheelFilename::from_str(&filename)?; - let compatible_tags = CompatibleTags::current(location.get_python_version())?; - filename.compatibility(&compatible_tags)?; - - install_wheel( - &locked_dir, - File::open(wheel)?, - filename, - false, - true, - &[], - // Only relevant for monotrail style installation - "", - interpreter, - ) -} diff --git a/crates/install-wheel-rs/src/main.rs b/crates/install-wheel-rs/src/main.rs index e45183e75..427328bf1 100644 --- a/crates/install-wheel-rs/src/main.rs +++ b/crates/install-wheel-rs/src/main.rs @@ -1,10 +1,11 @@ use clap::Parser; use fs_err::File; -use install_wheel_rs::{install_wheel, CompatibleTags, Error, InstallLocation, WheelFilename}; +use install_wheel_rs::{install_wheel, Error, InstallLocation}; #[cfg(feature = "rayon")] use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::path::PathBuf; use std::str::FromStr; +use wheel_filename::WheelFilename; /// Low level install CLI, mainly used for testing #[derive(Parser)] @@ -45,8 +46,6 @@ fn main() -> Result<(), Error> { .ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))? .to_string_lossy(); let filename = WheelFilename::from_str(&filename)?; - let compatible_tags = CompatibleTags::current(location.get_python_version())?; - filename.compatibility(&compatible_tags)?; Ok((wheel, filename)) }) .collect::>()?; @@ -66,7 +65,7 @@ fn main() -> Result<(), Error> { install_wheel( &locked_dir, File::open(wheel)?, - filename, + &filename, args.compile, !args.skip_hashes, &[], diff --git a/crates/install-wheel-rs/src/python_bindings.rs b/crates/install-wheel-rs/src/python_bindings.rs index edaa20684..50384e7fd 100644 --- a/crates/install-wheel-rs/src/python_bindings.rs +++ b/crates/install-wheel-rs/src/python_bindings.rs @@ -1,6 +1,6 @@ #![allow(clippy::format_push_string)] // I will not replace clear and infallible with fallible, io looking code -use crate::{install_wheel, CompatibleTags, Error, InstallLocation, LockedDir, WheelFilename}; +use crate::{install_wheel, Error, InstallLocation, LockedDir}; use pyo3::create_exception; use pyo3::types::PyModule; use pyo3::{pyclass, pymethods, pymodule, PyErr, PyResult, Python}; @@ -8,6 +8,7 @@ use std::env; use std::fs::File; use std::path::{Path, PathBuf}; use std::str::FromStr; +use wheel_filename::WheelFilename; create_exception!( install_wheel_rs, @@ -17,11 +18,11 @@ create_exception!( impl From for PyErr { fn from(err: Error) -> Self { - let mut accumulator = format!("Failed to install wheels: {}", err); + let mut accumulator = format!("Failed to install wheels: {err}"); let mut current_err: &dyn std::error::Error = &err; while let Some(cause) = current_err.source() { - accumulator.push_str(&format!("\n Caused by: {}", cause)); + accumulator.push_str(&format!("\n Caused by: {cause}")); current_err = cause; } PyWheelInstallerError::new_err(accumulator) @@ -36,7 +37,8 @@ struct LockedVenv { #[pymethods] impl LockedVenv { #[new] - pub fn new(py: Python, venv: PathBuf) -> PyResult { + #[allow(clippy::needless_pass_by_value)] + pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult { Ok(Self { location: InstallLocation::Venv { venv_base: LockedDir::acquire(&venv)?, @@ -45,7 +47,7 @@ impl LockedVenv { }) } - pub fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> { + pub(crate) fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> { // Would be nicer through https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath let sys_executable: String = py.import("sys")?.getattr("executable")?.extract()?; @@ -56,13 +58,10 @@ impl LockedVenv { .ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))? .to_string_lossy(); let filename = WheelFilename::from_str(&filename)?; - let compatible_tags = CompatibleTags::current(self.location.get_python_version())?; - filename.compatibility(&compatible_tags)?; - install_wheel( &self.location, File::open(wheel)?, - filename, + &filename, true, true, &[], @@ -76,7 +75,7 @@ impl LockedVenv { } #[pymodule] -pub fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> { +pub(crate) fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> { // Good enough for now if env::var_os("RUST_LOG").is_some() { tracing_subscriber::fmt::init(); diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index f4274d6bf..c83b1f207 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -1,8 +1,12 @@ #![allow(clippy::needless_borrow)] -use crate::install_location::{InstallLocation, LockedDir}; -use crate::wheel_tags::WheelFilename; -use crate::{normalize_name, Error}; +use std::collections::{HashMap, HashSet}; +use std::ffi::OsString; +use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Stdio}; +use std::{env, io, iter}; + use configparser::ini::Ini; use data_encoding::BASE64URL_NOPAD; use fs_err as fs; @@ -11,12 +15,6 @@ use mailparse::MailHeaderMap; use regex::Regex; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use std::collections::{HashMap, HashSet}; -use std::ffi::OsString; -use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write}; -use std::path::{Path, PathBuf}; -use std::process::{Command, ExitStatus, Stdio}; -use std::{env, io, iter}; use tempfile::{tempdir, TempDir}; use tracing::{debug, error, span, warn, Level}; use walkdir::WalkDir; @@ -24,12 +22,17 @@ use zip::result::ZipError; use zip::write::FileOptions; use zip::{ZipArchive, ZipWriter}; +use wheel_filename::WheelFilename; + +use crate::install_location::{InstallLocation, LockedDir}; +use crate::{normalize_name, Error}; + /// `#!/usr/bin/env python` pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python"; -pub const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe"); -pub const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe"); -pub const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe"); +pub(crate) const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe"); +pub(crate) const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe"); +pub(crate) const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe"); /// Line in a RECORD file /// @@ -43,15 +46,16 @@ pub struct RecordEntry { pub path: String, pub hash: Option, #[allow(dead_code)] - pub size: Option, + pub size: Option, } -/// Minimal direct_url.json schema +/// Minimal `direct_url.json` schema /// /// /// #[derive(Serialize)] struct DirectUrl { + #[allow(clippy::zero_sized_map_values)] archive_info: HashMap<(), ()>, url: String, } @@ -95,7 +99,7 @@ impl Script { let captures = script_regex .captures(value) - .ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{}'", value)))?; + .ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{value}'")))?; if let Some(script_extras) = captures.name("extras") { let script_extras = script_extras .as_str() @@ -130,10 +134,7 @@ from {module} import {import_name} if __name__ == "__main__": sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) sys.exit({import_name}()) -"##, - shebang = shebang, - module = module, - import_name = import_name +"## ) } @@ -144,7 +145,7 @@ fn read_scripts_from_section( extras: Option<&[String]>, ) -> Result, Error> { let mut scripts = Vec::new(); - for (script_name, python_location) in scripts_section.iter() { + for (script_name, python_location) in scripts_section { match python_location { Some(value) => { if let Some(script) = Script::from_value(script_name, value, extras)? { @@ -153,8 +154,7 @@ fn read_scripts_from_section( } None => { return Err(Error::InvalidWheel(format!( - "[{}] key {} must have a value", - section_name, script_name + "[{section_name}] key {script_name} must have a value" ))); } } @@ -162,9 +162,9 @@ fn read_scripts_from_section( Ok(scripts) } -/// Parses the entry_points.txt entry in the wheel for console scripts +/// Parses the `entry_points.txt` entry in the wheel for console scripts /// -/// Returns (script_name, module, function) +/// Returns (`script_name`, module, function) /// /// Extras are supposed to be ignored, which happens if you pass None for extras fn parse_scripts( @@ -177,9 +177,9 @@ fn parse_scripts( Ok(mut file) => { let mut ini_text = String::new(); file.read_to_string(&mut ini_text)?; - Ini::new_cs().read(ini_text).map_err(|err| { - Error::InvalidWheel(format!("entry_points.txt is invalid: {}", err)) - })? + Ini::new_cs() + .read(ini_text) + .map_err(|err| Error::InvalidWheel(format!("entry_points.txt is invalid: {err}")))? } Err(ZipError::FileNotFound) => return Ok((Vec::new(), Vec::new())), Err(err) => return Err(Error::from_zip_error(entry_points_path, err)), @@ -204,7 +204,10 @@ fn parse_scripts( /// /// which in turn got it from std /// -pub fn copy_and_hash(reader: &mut impl Read, writer: &mut impl Write) -> io::Result<(u64, String)> { +pub(crate) fn copy_and_hash( + reader: &mut impl Read, + writer: &mut impl Write, +) -> io::Result<(u64, String)> { // TODO: Do we need to support anything besides sha256? let mut hasher = Sha256::new(); // Same buf size as std. Note that this number is important for performance @@ -361,7 +364,7 @@ fn get_shebang(location: &InstallLocation) -> String { } else { path }; - format!("#!{}", path) + format!("#!{path}") } else { // This will use the monotrail binary moonlighting as python. `python` alone doesn't, // we need env to find the python link we put in PATH @@ -369,14 +372,14 @@ fn get_shebang(location: &InstallLocation) -> String { } } -/// Ported from https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262 -/// /// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual /// python after it. /// /// TODO pyw scripts /// /// TODO: a nice, reproducible-without-distlib rust solution +/// +/// fn windows_script_launcher(launcher_python_script: &str) -> Result, Error> { let launcher_bin = match env::consts::ARCH { "x84" => LAUNCHER_T32, @@ -384,9 +387,8 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result, Erro "aarch64" => LAUNCHER_T64_ARM, arch => { let error = format!( - "Don't know how to create windows launchers for script for {}, \ - only x86, x86_64 and aarch64 (64-bit arm) are supported", - arch + "Don't know how to create windows launchers for script for {arch}, \ + only x86, x86_64 and aarch64 (64-bit arm) are supported" ); return Err(Error::OsVersionDetection(error)); } @@ -414,7 +416,7 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result, Erro /// Create the wrapper scripts in the bin folder of the venv for launching console scripts /// -/// We also pass venv_base so we can write the same path as pip does +/// We also pass `venv_base` so we can write the same path as pip does /// /// TODO: Test for this launcher directly in install-wheel-rs fn write_script_entrypoints( @@ -486,16 +488,13 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> { // {distribution}-{version}.dist-info/WHEEL is metadata about the archive itself in the same basic key: value format: let data = parse_key_value_file(&mut wheel_text.as_bytes(), "WHEEL")?; - let wheel_version = if let Some(wheel_version) = - data.get("Wheel-Version").and_then(|wheel_versions| { - if let [wheel_version] = wheel_versions.as_slice() { - wheel_version.split_once('.') - } else { - None - } - }) { - wheel_version - } else { + let Some(wheel_version) = data.get("Wheel-Version").and_then(|wheel_versions| { + if let [wheel_version] = wheel_versions.as_slice() { + wheel_version.split_once('.') + } else { + None + } + }) else { return Err(Error::InvalidWheel( "Invalid Wheel-Version in WHEEL file".to_string(), )); @@ -515,7 +514,7 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> { ))); } if wheel_version.1 > "0" { - eprint!( + warn!( "Warning: Unsupported wheel minor version (expected {}, got {})", 0, wheel_version.1 ); @@ -553,18 +552,15 @@ fn bytecode_compile( retries -= 1; if status.success() || retries == 0 { break (status, lines); - } else { - warn!( - "Failed to compile {} with python compileall, retrying", - name, - ); } + + warn!("Failed to compile {name} with python compileall, retrying",); }; if !status.success() { // lossy because we want the error reporting to survive c̴̞̏ü̸̜̹̈́ŕ̴͉̈ś̷̤ė̵̤͋d̷͙̄ filenames in the zip return Err(Error::PythonSubcommand(io::Error::new( io::ErrorKind::Other, - format!("Failed to run python compileall, log above: {}", status), + format!("Failed to run python compileall, log above: {status}"), ))); } @@ -606,7 +602,7 @@ fn bytecode_compile( path: pyc_path.display().to_string(), hash: None, size: None, - }) + }); } Ok(()) @@ -655,7 +651,7 @@ fn bytecode_compile_inner( let line = line.map_err(|err| { Error::PythonSubcommand(io::Error::new( io::ErrorKind::Other, - format!("Invalid utf-8 returned by python compileall: {}", err), + format!("Invalid utf-8 returned by python compileall: {err}"), )) })?; lines.push(line); @@ -671,17 +667,16 @@ fn bytecode_compile_inner( /// /// lib/python/site-packages/foo/__init__.py and lib/python/site-packages -> foo/__init__.py /// lib/marker.txt and lib/python/site-packages -> ../../marker.txt -/// bin/foo_launcher and lib/python/site-packages -> ../../../bin/foo_launcher +/// `bin/foo_launcher` and lib/python/site-packages -> ../../../`bin/foo_launcher` pub fn relative_to(path: &Path, base: &Path) -> Result { // Find the longest common prefix, and also return the path stripped from that prefix let (stripped, common_prefix) = base .ancestors() - .filter_map(|ancestor| { + .find_map(|ancestor| { path.strip_prefix(ancestor) .ok() .map(|stripped| (stripped, ancestor)) }) - .next() .ok_or_else(|| { Error::IO(io::Error::new( io::ErrorKind::Other, @@ -750,7 +745,7 @@ fn move_folder_recorded( fn install_script( site_packages: &Path, record: &mut [RecordEntry], - file: DirEntry, + file: &DirEntry, location: &InstallLocation, ) -> Result<(), Error> { let path = file.path(); @@ -818,7 +813,7 @@ fn install_script( })?; entry.path = target_path.display().to_string(); if let Some((size, encoded_hash)) = size_and_encoded_hash { - entry.size = Some(size as usize); + entry.size = Some(size); entry.hash = Some(encoded_hash); } Ok(()) @@ -863,7 +858,7 @@ fn install_data( continue; } - install_script(site_packages, record, file, &location)?; + install_script(site_packages, record, &file, &location)?; } } Some("headers") => { @@ -911,12 +906,12 @@ fn write_file_recorded( record.push(RecordEntry { path: relative_path.display().to_string(), hash: Some(encoded_hash), - size: Some(content.as_ref().len()), + size: Some(content.as_ref().len() as u64), }); Ok(()) } -/// Adds INSTALLER, REQUESTED and direct_url.json to the .dist-info dir +/// Adds INSTALLER, REQUESTED and `direct_url.json` to the .dist-info dir fn extra_dist_info( site_packages: &Path, dist_info_prefix: &str, @@ -983,7 +978,7 @@ pub fn parse_key_value_file( file: &mut impl Read, debug_filename: &str, ) -> Result>, Error> { - let mut data = HashMap::new(); + let mut data: HashMap> = HashMap::new(); let file = BufReader::new(file); for (line_no, line) in file.lines().enumerate() { @@ -993,13 +988,12 @@ pub fn parse_key_value_file( } let (key, value) = line.split_once(": ").ok_or_else(|| { Error::InvalidWheel(format!( - "Line {} of the {} file is invalid", - line_no, debug_filename + "Line {line_no} of the {debug_filename} file is invalid" )) })?; data.entry(key.to_string()) - .or_insert_with(Vec::new) - .push(value.to_string()) + .or_default() + .push(value.to_string()); } Ok(data) } @@ -1011,10 +1005,11 @@ pub fn parse_key_value_file( /// /// /// Wheel 1.0: +#[allow(clippy::too_many_arguments)] pub fn install_wheel( location: &InstallLocation, reader: impl Read + Seek, - filename: WheelFilename, + filename: &WheelFilename, compile: bool, check_hashes: bool, // initially used to the console scripts, currently unused. Keeping it because we likely need @@ -1181,11 +1176,11 @@ pub fn install_wheel( Ok(filename.get_tag()) } -/// From https://github.com/PyO3/python-pkginfo-rs -/// /// The metadata name may be uppercase, while the wheel and dist info names are lowercase, or /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase. /// Either way, we just search the wheel for the name +/// +/// fn find_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, @@ -1205,7 +1200,7 @@ fn find_dist_info( "Missing .dist-info directory".to_string(), )) } - [dist_info] => dist_info.to_string(), + [dist_info] => (*dist_info).to_string(), _ => { return Err(Error::InvalidWheel(format!( "Multiple .dist-info directories: {}", @@ -1216,7 +1211,7 @@ fn find_dist_info( Ok(dist_info) } -/// Adapted from https://github.com/PyO3/python-pkginfo-rs +/// fn read_metadata( dist_info_prefix: &str, archive: &mut ZipArchive, @@ -1231,50 +1226,45 @@ fn read_metadata( let mut mail = b"Content-Type: text/plain; charset=utf-8\n".to_vec(); mail.extend_from_slice(&content); let msg = mailparse::parse_mail(&mail) - .map_err(|err| Error::InvalidWheel(format!("Invalid {}: {}", metadata_file, err)))?; + .map_err(|err| Error::InvalidWheel(format!("Invalid {metadata_file}: {err}")))?; let headers = msg.get_headers(); let metadata_version = headers .get_first_value("Metadata-Version") .ok_or(Error::InvalidWheel(format!( - "No Metadata-Version field in {}", - metadata_file + "No Metadata-Version field in {metadata_file}" )))?; // Crude but it should do https://packaging.python.org/en/latest/specifications/core-metadata/#metadata-version // At time of writing: // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. if !(metadata_version.starts_with("1.") || metadata_version.starts_with("2.")) { return Err(Error::InvalidWheel(format!( - "Metadata-Version field has unsupported value {}", - metadata_version + "Metadata-Version field has unsupported value {metadata_version}" ))); } let name = headers .get_first_value("Name") .ok_or(Error::InvalidWheel(format!( - "No Name field in {}", - metadata_file + "No Name field in {metadata_file}" )))?; let version = headers .get_first_value("Version") .ok_or(Error::InvalidWheel(format!( - "No Version field in {}", - metadata_file + "No Version field in {metadata_file}" )))?; Ok((name, version)) } #[cfg(test)] mod test { - use super::parse_wheel_version; - use crate::wheel::{read_record_file, relative_to}; - use crate::{install_wheel, parse_key_value_file, InstallLocation, Script, WheelFilename}; - use fs_err as fs; + use std::path::Path; + use indoc::{formatdoc, indoc}; - use std::fs::File; - use std::path::{Path, PathBuf}; - use std::str::FromStr; - use tempfile::TempDir; + + use crate::wheel::{read_record_file, relative_to}; + use crate::{parse_key_value_file, Script}; + + use super::parse_wheel_version; #[test] fn test_parse_key_value_file() { @@ -1331,58 +1321,6 @@ mod test { assert_eq!(expected, actual); } - /// Previously `__pycache__` paths were erroneously absolute - #[test] - fn installed_paths_relative() { - let filename = "colander-0.9.9-py2.py3-none-any.whl"; - let wheel = Path::new("../../test-data/wheels").join(filename); - let temp_dir = TempDir::new().unwrap(); - // TODO: Would be nicer to pick the default python here, but i don't want to launch a - // subprocess - let python = if cfg!(target_os = "windows") { - PathBuf::from("python.exe") - } else { - PathBuf::from("python3.8") - }; - let install_location = InstallLocation::::Monotrail { - monotrail_root: temp_dir.path().to_path_buf(), - python: python.clone(), - python_version: (3, 8), - } - .acquire_lock() - .unwrap(); - install_wheel( - &install_location, - File::open(wheel).unwrap(), - WheelFilename::from_str(&filename).unwrap(), - true, - true, - &[], - "0.9.9", - &python, - ) - .unwrap(); - - let base = temp_dir - .path() - .join("colander") - .join("0.9.9") - .join("py2.py3-none-any"); - let mid = if cfg!(windows) { - base.join("Lib") - } else { - base.join("lib").join("python") - }; - let record = mid - .join("site-packages") - .join("colander-0.9.9.dist-info") - .join("RECORD"); - let record = fs::read_to_string(&record).unwrap(); - for line in record.lines() { - assert!(!line.starts_with('/'), "{}", line); - } - } - #[test] fn test_relative_to() { assert_eq!( diff --git a/crates/install-wheel-rs/src/wheel_tags.rs b/crates/install-wheel-rs/src/wheel_tags.rs deleted file mode 100644 index b4a06ec6d..000000000 --- a/crates/install-wheel-rs/src/wheel_tags.rs +++ /dev/null @@ -1,937 +0,0 @@ -//! Parses the wheel filename, the current host os/arch and checks wheels for compatibility - -use crate::Error; -use fs_err as fs; -use goblin::elf::Elf; -use once_cell::sync::Lazy; -use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI}; -use regex::Regex; -use serde::Deserialize; -use std::fmt; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str::FromStr; -use tracing::trace; - -/// The name of a wheel split into its parts ([PEP 491](https://peps.python.org/pep-0491/)) -/// -/// Ignores the build tag atm. -/// -/// ``` -/// use std::str::FromStr; -/// use install_wheel_rs::WheelFilename; -/// -/// let filename = WheelFilename::from_str("foo-1.0-py32-none-any.whl").unwrap(); -/// assert_eq!(filename, WheelFilename { -/// distribution: "foo".to_string(), -/// version: "1.0".to_string(), -/// python_tag: vec!["py32".to_string()], -/// abi_tag: vec!["none".to_string()], -/// platform_tag: vec!["any".to_string()] -/// }); -/// let filename = WheelFilename::from_str( -/// "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" -/// ).unwrap(); -/// assert_eq!(filename, WheelFilename { -/// distribution: "numpy".to_string(), -/// version: "1.26.0".to_string(), -/// python_tag: vec!["cp312".to_string()], -/// abi_tag: vec!["cp312".to_string()], -/// platform_tag: vec![ -/// "manylinux_2_17_aarch64".to_string(), -/// "manylinux2014_aarch64".to_string() -/// ] -/// }); -/// ``` -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct WheelFilename { - pub distribution: String, - pub version: String, - pub python_tag: Vec, - pub abi_tag: Vec, - pub platform_tag: Vec, -} - -impl FromStr for WheelFilename { - type Err = Error; - - fn from_str(filename: &str) -> Result { - let basename = filename.strip_suffix(".whl").ok_or_else(|| { - Error::InvalidWheelFileName(filename.to_string(), "Must end with .whl".to_string()) - })?; - // https://www.python.org/dev/peps/pep-0427/#file-name-convention - match basename.split('-').collect::>().as_slice() { - // TODO: Build tag precedence - &[distribution, version, _, python_tag, abi_tag, platform_tag] - | &[distribution, version, python_tag, abi_tag, platform_tag] => Ok(WheelFilename { - distribution: distribution.to_string(), - version: version.to_string(), - python_tag: python_tag.split('.').map(String::from).collect(), - abi_tag: abi_tag.split('.').map(String::from).collect(), - platform_tag: platform_tag.split('.').map(String::from).collect(), - }), - _ => Err(Error::InvalidWheelFileName( - filename.to_string(), - "Expected four or five dashes (\"-\") in the filename".to_string(), - )), - } - } -} - -impl WheelFilename { - /// Returns Some(precedence) is the wheels are compatible, otherwise none - /// - /// Precedence is e.g. used to install newer manylinux wheels over older manylinux wheels - pub fn compatibility(&self, compatible_tags: &CompatibleTags) -> Result { - compatible_tags - .iter() - .enumerate() - .filter_map(|(precedence, tag)| { - if self.python_tag.contains(&tag.0) - && self.abi_tag.contains(&tag.1) - && self.platform_tag.contains(&tag.2) - { - Some(precedence) - } else { - None - } - }) - .next() - .ok_or_else(|| Error::IncompatibleWheel { - os: compatible_tags.os.clone(), - arch: compatible_tags.arch, - }) - } - - /// Effectively undoes the wheel filename parsing step - pub fn get_tag(&self) -> String { - format!( - "{}-{}-{}", - self.python_tag.join("."), - self.abi_tag.join("."), - self.platform_tag.join(".") - ) - } -} - -/// A platform, defined by the list of compatible wheel tags in order -pub struct CompatibleTags { - pub os: Os, - pub arch: Arch, - pub tags: Vec<(String, String, String)>, -} - -impl Deref for CompatibleTags { - type Target = [(String, String, String)]; - - fn deref(&self) -> &Self::Target { - &self.tags - } -} - -/// Returns the compatible tags in a (python_tag, abi_tag, platform_tag) format, ordered from -/// highest precedence to lowest precedence -impl CompatibleTags { - /// Compatible tags for the current operating system and architecture - pub fn current(python_version: (u8, u8)) -> Result { - Self::new(python_version, Os::current()?, Arch::current()?) - } - - pub fn new(python_version: (u8, u8), os: Os, arch: Arch) -> Result { - assert_eq!(python_version.0, 3); - let mut tags = Vec::new(); - let platform_tags = compatible_platform_tags(&os, &arch)?; - // 1. This exact c api version - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - format!( - "cp{}{}{}", - python_version.0, - python_version.1, - // hacky but that's legacy anyways - if python_version.1 <= 7 { "m" } else { "" } - ), - platform_tag.clone(), - )); - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - "none".to_string(), - platform_tag.clone(), - )); - } - // 2. abi3 and no abi (e.g. executable binary) - // For some reason 3.2 is the minimum python for the cp abi - for minor in 2..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, minor), - "abi3".to_string(), - platform_tag.clone(), - )); - } - } - // 3. no abi (e.g. executable binary) - for minor in 0..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - platform_tag.clone(), - )); - } - } - // 4. major only - for platform_tag in platform_tags { - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - platform_tag, - )); - } - // 5. no binary - for minor in 0..=python_version.1 { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - "any".to_string(), - )); - } - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - "any".to_string(), - )); - Ok(CompatibleTags { os, arch, tags }) - } -} - -/// All supported operating system -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum Os { - Manylinux { major: u16, minor: u16 }, - Musllinux { major: u16, minor: u16 }, - Windows, - Macos { major: u16, minor: u16 }, - FreeBsd { release: String }, - NetBsd { release: String }, - OpenBsd { release: String }, - Dragonfly { release: String }, - Illumos { release: String, arch: String }, - Haiku { release: String }, -} - -impl Os { - fn detect_linux_libc() -> Result { - let libc = find_libc()?; - let linux = if let Ok(Some((major, minor))) = get_musl_version(&libc) { - Os::Musllinux { major, minor } - } else if let Ok(glibc_ld) = fs::read_link(&libc) { - // Try reading the link first as it's faster - let filename = glibc_ld - .file_name() - .ok_or_else(|| { - Error::OsVersionDetection("Expected the glibc ld to be a file".to_string()) - })? - .to_string_lossy(); - #[allow(non_upper_case_globals)] - static expr: Lazy = - Lazy::new(|| Regex::new(r"ld-(\d{1,3})\.(\d{1,3})\.so").unwrap()); - - if let Some(capture) = expr.captures(&filename) { - let major = capture.get(1).unwrap().as_str().parse::().unwrap(); - let minor = capture.get(2).unwrap().as_str().parse::().unwrap(); - Os::Manylinux { major, minor } - } else { - trace!("Couldn't use ld filename, using `ldd --version`"); - // runs `ldd --version` - let version = glibc_version::get_version().map_err(|err| { - Error::OsVersionDetection(format!( - "Failed to determine glibc version with `ldd --version`: {}", - err - )) - })?; - Os::Manylinux { - major: version.major as u16, - minor: version.minor as u16, - } - } - } else { - return Err(Error::OsVersionDetection("Couldn't detect neither glibc version nor musl libc version, at least one of which is required".to_string())); - }; - trace!("libc: {}", linux); - Ok(linux) - } - - pub fn current() -> Result { - let target_triple = target_lexicon::HOST; - - let os = match target_triple.operating_system { - target_lexicon::OperatingSystem::Linux => Self::detect_linux_libc()?, - target_lexicon::OperatingSystem::Windows => Os::Windows, - target_lexicon::OperatingSystem::MacOSX { major, minor, .. } => { - Os::Macos { major, minor } - } - target_lexicon::OperatingSystem::Darwin => { - let (major, minor) = get_mac_os_version()?; - Os::Macos { major, minor } - } - target_lexicon::OperatingSystem::Netbsd => Os::NetBsd { - release: PlatformInfo::new() - .map_err(Error::PlatformInfo)? - .release() - .to_string_lossy() - .to_string(), - }, - target_lexicon::OperatingSystem::Freebsd => Os::FreeBsd { - release: PlatformInfo::new() - .map_err(Error::PlatformInfo)? - .release() - .to_string_lossy() - .to_string(), - }, - target_lexicon::OperatingSystem::Openbsd => Os::OpenBsd { - release: PlatformInfo::new() - .map_err(Error::PlatformInfo)? - .release() - .to_string_lossy() - .to_string(), - }, - target_lexicon::OperatingSystem::Dragonfly => Os::Dragonfly { - release: PlatformInfo::new() - .map_err(Error::PlatformInfo)? - .release() - .to_string_lossy() - .to_string(), - }, - target_lexicon::OperatingSystem::Illumos => { - let platform_info = PlatformInfo::new().map_err(Error::PlatformInfo)?; - Os::Illumos { - release: platform_info.release().to_string_lossy().to_string(), - arch: platform_info.machine().to_string_lossy().to_string(), - } - } - target_lexicon::OperatingSystem::Haiku => Os::Haiku { - release: PlatformInfo::new() - .map_err(Error::PlatformInfo)? - .release() - .to_string_lossy() - .to_string(), - }, - unsupported => { - return Err(Error::OsVersionDetection(format!( - "The operating system {:?} is not supported", - unsupported - ))) - } - }; - Ok(os) - } -} - -impl fmt::Display for Os { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Os::Manylinux { .. } => write!(f, "Manylinux"), - Os::Musllinux { .. } => write!(f, "Musllinux"), - Os::Windows => write!(f, "Windows"), - Os::Macos { .. } => write!(f, "MacOS"), - Os::FreeBsd { .. } => write!(f, "FreeBSD"), - Os::NetBsd { .. } => write!(f, "NetBSD"), - Os::OpenBsd { .. } => write!(f, "OpenBSD"), - Os::Dragonfly { .. } => write!(f, "DragonFly"), - Os::Illumos { .. } => write!(f, "Illumos"), - Os::Haiku { .. } => write!(f, "Haiku"), - } - } -} - -/// All supported CPU architectures -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum Arch { - Aarch64, - Armv7L, - Powerpc64Le, - Powerpc64, - X86, - X86_64, - S390X, -} - -impl fmt::Display for Arch { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Arch::Aarch64 => write!(f, "aarch64"), - Arch::Armv7L => write!(f, "armv7l"), - Arch::Powerpc64Le => write!(f, "ppc64le"), - Arch::Powerpc64 => write!(f, "ppc64"), - Arch::X86 => write!(f, "i686"), - Arch::X86_64 => write!(f, "x86_64"), - Arch::S390X => write!(f, "s390x"), - } - } -} - -impl Arch { - pub fn current() -> Result { - let target_triple = target_lexicon::HOST; - let arch = match target_triple.architecture { - target_lexicon::Architecture::X86_64 => Arch::X86_64, - target_lexicon::Architecture::X86_32(_) => Arch::X86, - target_lexicon::Architecture::Arm(_) => Arch::Armv7L, - target_lexicon::Architecture::Aarch64(_) => Arch::Aarch64, - target_lexicon::Architecture::Powerpc64 => Arch::Powerpc64, - target_lexicon::Architecture::Powerpc64le => Arch::Powerpc64Le, - target_lexicon::Architecture::S390x => Arch::S390X, - unsupported => { - return Err(Error::OsVersionDetection(format!( - "The architecture {} is not supported", - unsupported - ))); - } - }; - Ok(arch) - } - - /// Returns the oldest possible Manylinux tag for this architecture - pub fn get_minimum_manylinux_minor(&self) -> u16 { - match self { - // manylinux 2014 - Arch::Aarch64 | Arch::Armv7L | Arch::Powerpc64 | Arch::Powerpc64Le | Arch::S390X => 17, - // manylinux 1 - Arch::X86 | Arch::X86_64 => 5, - } - } -} - -fn get_mac_os_version() -> Result<(u16, u16), Error> { - // This is actually what python does - // https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428 - #[derive(Deserialize)] - #[serde(rename_all = "PascalCase")] - struct SystemVersion { - product_version: String, - } - let system_version: SystemVersion = - plist::from_file("/System/Library/CoreServices/SystemVersion.plist") - .map_err(|err| Error::OsVersionDetection(err.to_string()))?; - - let invalid_mac_os_version = || { - Error::OsVersionDetection(format!( - "Invalid mac os version {}", - system_version.product_version - )) - }; - match system_version - .product_version - .split('.') - .collect::>() - .as_slice() - { - [major, minor] | [major, minor, _] => { - let major = major.parse::().map_err(|_| invalid_mac_os_version())?; - let minor = minor.parse::().map_err(|_| invalid_mac_os_version())?; - Ok((major, minor)) - } - _ => Err(invalid_mac_os_version()), - } -} - -/// Determine the appropriate binary formats for a mac os version. -/// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314 -fn get_mac_binary_formats(major: u16, minor: u16, arch: &Arch) -> Vec { - let mut formats = vec![match arch { - Arch::Aarch64 => "arm64".to_string(), - _ => arch.to_string(), - }]; - - if matches!(arch, Arch::X86_64) { - if (major, minor) < (10, 4) { - return vec![]; - } - formats.extend([ - "intel".to_string(), - "fat64".to_string(), - "fat32".to_string(), - ]); - } - - if matches!(arch, Arch::X86_64 | Arch::Aarch64) { - formats.push("universal2".to_string()); - } - - if matches!(arch, Arch::X86_64) { - formats.push("universal".to_string()); - } - - formats -} - -/// Find musl libc path from executable's ELF header -pub fn find_libc() -> Result { - let buffer = fs::read("/bin/ls")?; - let error_str = "Couldn't parse /bin/ls for detecting the ld version"; - let elf = Elf::parse(&buffer) - .map_err(|err| Error::OsVersionDetection(format!("{}: {}", error_str, err)))?; - if let Some(elf_interpreter) = elf.interpreter { - Ok(PathBuf::from(elf_interpreter)) - } else { - Err(Error::OsVersionDetection(error_str.to_string())) - } -} - -/// Read the musl version from libc library's output. Taken from maturin -/// -/// The libc library should output something like this to stderr:: -/// -/// musl libc (x86_64) -/// Version 1.2.2 -/// Dynamic Program Loader -pub fn get_musl_version(ld_path: impl AsRef) -> std::io::Result> { - let output = Command::new(ld_path.as_ref()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .output()?; - let stderr = String::from_utf8_lossy(&output.stderr); - - #[allow(non_upper_case_globals)] - static expr: Lazy = Lazy::new(|| Regex::new(r"Version (\d{2,4})\.(\d{2,4})").unwrap()); - if let Some(capture) = expr.captures(&stderr) { - let major = capture.get(1).unwrap().as_str().parse::().unwrap(); - let minor = capture.get(2).unwrap().as_str().parse::().unwrap(); - return Ok(Some((major, minor))); - } - Ok(None) -} - -/// Returns the compatible platform tags from highest precedence to lowest precedence -/// -/// Examples: manylinux_2_17, macosx_11_0_arm64, win_amd64 -/// -/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2) -/// and "any". -/// -/// Bit of a mess, needs to be cleaned up. The order also isn't exactly matching that of pip yet, -/// but works good enough in practice -pub fn compatible_platform_tags(os: &Os, arch: &Arch) -> Result, Error> { - let platform_tags = match (os.clone(), *arch) { - (Os::Manylinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - // Use newer manylinux first like pip does - platform_tags.extend( - (arch.get_minimum_manylinux_minor()..=minor) - .rev() - .map(|minor| format!("manylinux_{}_{}_{}", major, minor, arch)), - ); - if (arch.get_minimum_manylinux_minor()..=minor).contains(&17) { - platform_tags.push(format!("manylinux2014_{}", arch)) - } - if (arch.get_minimum_manylinux_minor()..=minor).contains(&12) { - platform_tags.push(format!("manylinux2010_{}", arch)) - } - if (arch.get_minimum_manylinux_minor()..=minor).contains(&5) { - platform_tags.push(format!("manylinux1_{}", arch)) - } - platform_tags - } - (Os::Musllinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - // musl 1.1 is the lowest supported version in musllinux - platform_tags.extend( - (1..=minor) - .rev() - .map(|minor| format!("musllinux_{}_{}_{}", major, minor, arch)), - ); - platform_tags - } - (Os::Macos { major, minor }, Arch::X86_64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - match major { - 10 => { - // Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version - // number. The major version was always 10. - for minor in (0..=minor).rev() { - for binary_format in get_mac_binary_formats(major, minor, arch) { - platform_tags - .push(format!("macosx_{}_{}_{}", major, minor, binary_format)); - } - } - } - x if x >= 11 => { - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - for minor in (4..=16).rev() { - for binary_format in get_mac_binary_formats(10, minor, arch) { - platform_tags - .push(format!("macosx_{}_{}_{}", 10, minor, binary_format)); - } - } - } - _ => { - return Err(Error::OsVersionDetection(format!( - "Unsupported mac os version: {}", - major, - ))); - } - } - platform_tags - } - (Os::Macos { major, .. }, Arch::Aarch64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - platform_tags.extend( - (4..=16) - .rev() - .map(|minor| format!("macosx_{}_{}_universal2", 10, minor)), - ); - platform_tags - } - (Os::Windows, Arch::X86) => { - vec!["win32".to_string()] - } - (Os::Windows, Arch::X86_64) => { - vec!["win_amd64".to_string()] - } - (Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()], - ( - Os::FreeBsd { release: _ } - | Os::NetBsd { release: _ } - | Os::OpenBsd { release: _ } - | Os::Dragonfly { release: _ } - | Os::Haiku { release: _ }, - _, - ) => { - let info = PlatformInfo::new().map_err(Error::PlatformInfo)?; - let release = info.release().to_string_lossy().replace(['.', '-'], "_"); - vec![format!( - "{}_{}_{}", - os.to_string().to_lowercase(), - release, - arch - )] - } - ( - Os::Illumos { - mut release, - mut arch, - }, - _, - ) => { - let mut os = os.to_string().to_lowercase(); - // See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730 - if let Some((major, other)) = release.split_once('_') { - let major_ver: u64 = major.parse().map_err(|err| { - Error::OsVersionDetection(format!( - "illumos major version is not a number: {}", - err - )) - })?; - if major_ver >= 5 { - // SunOS 5 == Solaris 2 - os = "solaris".to_string(); - release = format!("{}_{}", major_ver - 3, other); - arch = format!("{}_64bit", arch); - } - } - vec![format!("{}_{}_{}", os, release, arch)] - } - _ => { - return Err(Error::OsVersionDetection(format!( - "Unsupported operating system and architecture combination: {} {}", - os, arch - ))); - } - }; - Ok(platform_tags) -} - -#[cfg(test)] -mod test { - use super::{compatible_platform_tags, WheelFilename}; - use crate::{Arch, CompatibleTags, Error, Os}; - use fs_err::File; - use std::str::FromStr; - - const FILENAMES: &[&str] = &[ - "numpy-1.22.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "numpy-1.22.2-cp310-cp310-win_amd64.whl", - "numpy-1.22.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "numpy-1.22.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "numpy-1.22.2-cp310-cp310-macosx_11_0_arm64.whl", - "numpy-1.22.2-cp310-cp310-macosx_10_14_x86_64.whl", - "numpy-1.22.2-cp39-cp39-win_amd64.whl", - "numpy-1.22.2-cp39-cp39-win32.whl", - "numpy-1.22.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "numpy-1.22.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "numpy-1.22.2-cp39-cp39-macosx_11_0_arm64.whl", - "numpy-1.22.2-cp39-cp39-macosx_10_14_x86_64.whl", - "numpy-1.22.2-cp38-cp38-win_amd64.whl", - "numpy-1.22.2-cp38-cp38-win32.whl", - "numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl", - "numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl", - "tqdm-4.62.3-py2.py3-none-any.whl", - ]; - - /// Test that we can parse the filenames - #[test] - fn test_wheel_filename_parsing() -> Result<(), Error> { - for filename in FILENAMES { - WheelFilename::from_str(filename)?; - } - Ok(()) - } - - /// Test that we correctly identify compatible pairs - #[test] - fn test_compatibility() -> Result<(), Error> { - let filenames = [ - ( - "numpy-1.22.2-cp38-cp38-win_amd64.whl", - ((3, 8), Os::Windows, Arch::X86_64), - ), - ( - "numpy-1.22.2-cp38-cp38-win32.whl", - ((3, 8), Os::Windows, Arch::X86), - ), - ( - "numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - ( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - ), - ), - ( - "numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - ( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::Aarch64, - ), - ), - ( - "numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl", - ( - (3, 8), - Os::Macos { - major: 11, - minor: 0, - }, - Arch::Aarch64, - ), - ), - ( - "numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl", - ( - (3, 8), - // Test backwards compatibility here - Os::Macos { - major: 11, - minor: 0, - }, - Arch::X86_64, - ), - ), - ( - "ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", - ( - (3, 8), - Os::Macos { - major: 12, - minor: 0, - }, - Arch::X86_64, - ), - ), - ( - "ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", - ( - (3, 8), - Os::Macos { - major: 12, - minor: 0, - }, - Arch::Aarch64, - ), - ), - ( - "tqdm-4.62.3-py2.py3-none-any.whl", - ( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - ), - ), - ]; - - for (filename, (python_version, os, arch)) in filenames { - let compatible_tags = CompatibleTags::new(python_version, os, arch)?; - assert!( - WheelFilename::from_str(filename)? - .compatibility(&compatible_tags) - .is_ok(), - "{}", - filename - ); - } - Ok(()) - } - - /// Test that incompatible pairs don't pass is_compatible - #[test] - fn test_compatibility_filter() -> Result<(), Error> { - let compatible_tags = CompatibleTags::new( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - )?; - - let compatible: Vec<&str> = FILENAMES - .iter() - .filter(|filename| { - WheelFilename::from_str(filename) - .unwrap() - .compatibility(&compatible_tags) - .is_ok() - }) - .cloned() - .collect(); - assert_eq!( - vec![ - "numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "tqdm-4.62.3-py2.py3-none-any.whl" - ], - compatible - ); - Ok(()) - } - - fn get_ubuntu_20_04_tags() -> Vec { - serde_json::from_reader(File::open("../../test-data/tags/cp38-ubuntu-20-04.json").unwrap()) - .unwrap() - } - - /// Check against the tags that packaging.tags reports as compatible - #[test] - fn ubuntu_20_04_compatible() -> Result<(), Error> { - let tags = get_ubuntu_20_04_tags(); - for tag in tags { - let compatible_tags = CompatibleTags::new( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - )?; - - assert!( - WheelFilename::from_str(&format!("foo-1.0-{}.whl", tag))? - .compatibility(&compatible_tags) - .is_ok(), - "{}", - tag - ) - } - Ok(()) - } - - /// Check against the tags that packaging.tags reports as compatible - #[test] - fn ubuntu_20_04_list() -> Result<(), Error> { - let expected_tags = get_ubuntu_20_04_tags(); - let actual_tags: Vec = CompatibleTags::new( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - )? - .iter() - .map(|(python_tag, abi_tag, platform_tag)| { - format!("{}-{}-{}", python_tag, abi_tag, platform_tag) - }) - .collect(); - assert_eq!(expected_tags, actual_tags); - Ok(()) - } - - #[test] - fn test_precedence() { - let tags = CompatibleTags::new( - (3, 8), - Os::Manylinux { - major: 2, - minor: 31, - }, - Arch::X86_64, - ) - .unwrap(); - let pairs = [ - ( - "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl" - ), - ( - "regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "regex-2022.10.31-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" - ), - ]; - for (higher_str, lower_str) in pairs { - let higher = WheelFilename::from_str(higher_str).unwrap(); - let lower = WheelFilename::from_str(lower_str).unwrap(); - let higher_precedence = higher.compatibility(&tags).unwrap(); - let lower_precedence = lower.compatibility(&tags).unwrap(); - assert!( - higher_precedence < lower_precedence, - "{} {} {} {}", - higher_str, - higher_precedence, - lower_str, - lower_precedence - ); - } - } - - /// Basic does-it-work test - #[test] - fn host_arch() -> Result<(), Error> { - let os = Os::current()?; - let arch = Arch::current()?; - compatible_platform_tags(&os, &arch)?; - Ok(()) - } -} diff --git a/crates/install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl b/crates/install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl deleted file mode 100644 index 56dc953103b9cbb5eed28a1684d63ac08a0f3aa0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5495 zcmaKw2T)V%w#P#W9i;c(YY5V-1_(V6dQlJs#84wODqTUED7{LNA|><|dQ*Bw5C~1W zpac~Risb>_Zlh536$35OBM0nLn}QXH2_LwB2dzh`va z!YwEA3oZ25z`_Ot19A8V_yew|?WxBt+0QS3lx&oyb6u&pd`(%CWQCho$V%ZO;>n4F z%bL=-r~!W>oZ|Lat!P)SV_QPxqTnh*hV&EL5w4eYev?sYJl|-H;J}=>&N^1p z6P}u8x(LK-D;0_YSMO%*CsO$8FUT2Gz0VY)L5sfuUa#J}9NHQi)Ps6c{m|K_=r%@H zP0QH)@uCZF9n*jdjeiWKllWK}zdG4{SLESIx9Ye>s+6Q%}t(F56mM&*$7$H5CQhE+jJX*5LdDh zjQ4I9bF@+R67$jJ2~0x=)sWKgA-yMz&lKL4DyFmhCruO}s<{Wt?iP3&r#pflYV!n-X1JYTL1^568 zdQ%$MMp)~6|MY96Xgt4Sq>%c3Ai0D&C)yrm)BTa?Hb^ZcwpYs=v=u2z7)R>K;L?R^ zzh6_ovgB3ObkJm##eu%+h*_}X127!acT|T^e!-%t^pTY<;!zro)rYp(3^u$$u3WcA z&@EBtfD^D*&U62P{!i>Q_{5HS4J;qR%O|B>m>8T#Hk8v>*fP7cVk^x+PfUfE!g^5IQMlG&0Ts_@dSujMNeH2-9d-7Whoeq zdbD(DiPBy3!QoD^J+Adtw~OTxg9I9<6aQMM=NK7$@sTRn+U~%`EmoO>DeliaIh>F>+Tz}qUc@R96XORS< zP^UT)MG1IOxB8EGRY8Q6cFk(@90!$8)Q$G_NS^v|_c5PTwUi&RC4@49NAyyJ{hdV{ z(%*H%qK>-VTXtS|TpuCx{?=@(axZprb_9wVNvkxNT5>u`8o{%F)gV!$d zW|CGx6M=}@Xf8$t)lQGl*fjDdi8t>;#h9r_2p{oXPCUHcw>^1jmOdFAN^U2zi+Po$ zlJPAj7NgN1UzM|myS*Y6X`WeAnOVhFk{T|&N1Sw8#aL{mJ~36dfS9THZjQ;J<|sgW z{HVccQH)4J@^pn}OEpTw4VuMV`eb{>7WBj?b8}0WW=e`aXjRW4h+S}GOx@Ir(^>D7 z=F2m4j56~klSt2;b(9|%|EfX#bUrwf1s=tcS(w$0v$i{DLF2|uo1Kwa!ZF)>9=_D> zQzV=H!a66=eLmr{w25K!G0*vofPLfrgDtBw1O8Ozr3BhO!e9>~pwANMNrXRfeF<5$>js8`Vt&=dE)UXJZ;=Yw3~zQw>B&FL&No9T;J za79C?oVbr8l^+NfBBkxh3A)TDKvWTBw*+ZgylBT z&katwYb=duv)H z({2$)wne;VBcDcid4bGjq+-Q9P;4_*H6Aew&jual9&%_Y1eCFct+|#vz!HPxP{8YJ z0btkh44TNOD}77vB@^m;Uf*0xd2Jy3`qE})SZqKSS+p!lcCA=kD#QqeS{RPZ(a5nXpR~9qd3f)>?9_vn)0NY(1H0dEeuC?5=T?L8uMxO)uKJY6{~9TVb;x%cR3mX z+dqb6)bo;uL)ya0as+x4?-)v~Rc#`Xz_OyrI1T)`ajy|`g{XccMm>=Fv^yZKR~ zsZp*#J07&5T9c0W9Q$6dP0U;@MH!}!vA<~=5$xHDx%O_6X&7>F_qNdXsEE1Uvs3vc z{6K?~^RVdHGviDKay?EZ#lsa*B=tDoHNDII?u6e==(F#sYpMuw`bBTxr;L*d_zGOa zVW5E0q?p1S3u(=a3rQ;{s!7HbPc}3d-C=p!e>26=C~p+}^YIjfKLsH~;RrXG_R-;XV>Q=EaZ;e} zeY(TfIiM*n`cv-fRKclvALo4vgA}#=iR%i7PEpKUZ$*rAb~#I)Czrd=67)M-J<;5kSU!rM7U=6&8#z=VBVO+=;> zjn(s-9>D}ptXOKZCxzRJBTKm4ro+qqO|(so0^iPObR;fwm37SSg{B5Op00}M-wDt= zq&fP5T>GXMBJQ|phGCl5wv^2S&lnqoKs+NumW_2%eHo*QbPD#RkESxV??8tjbq(t>5FdBs2ic;O&V11n$0OO&Tl~+Dap+x7j-cX>}9CHdyJk)FN3I#^GIOxzzSp{X~8&4v%-) z{_J*KhCiIt=$*Os>1}~61x2M6TP zM#UaHfW{rHQ6XAGPnp`$fa zJHwZFnkY!0m-X;l;jJ`mmhQ&X-k7FM_f~KPUgWLf?pc1)wVS1J#|k{_^Sn4Ukb@JR z#i?-Rm&{ISR|)(}>Jl9!d=^grv)3o9Fa8ZGx46d1XRMhXsrk(8AiO#(<-y6&uo`T) z+F-Jk1Qh039x;h!5L;X`{gm}0C^x0udCMT5VtLYR+?1&}taFWEPhKom6xy`wpdHFq z{WU_pS_pkF(Y*cobdjU8?PNA%)G$S_g4m|BRxU#Ot}`8N|G`0kJ9{q}StC3efA%I(EsbJh-H7AH+$9`2xH#S&{Zvan48-DC*#AZ#Jw(gI}-LF+JSmjmLB z9dA}hQMVGBuRE#f7&@rgkJx(#A{pkE`cr~fqH1_bJt&zCb?Gzg*X9`n$I2v6 zIQM&RU)3Ml4H4WkFM33zwxw-TRN$Lxc4_>I|A}ke{k9HIuncwMDEhUm|C0=DHN`P#tP88dlVh~Y08~YivK(wQBg0jw?tx-8rq2%32h{~yAavGsEoSmPXQzdyW z)6B%!->h=n1o+w0%S+gpVyWw$E1k5@X?Xm@02%6&V|RHwL!@R;nswFku0dz;*Dj5Y zU!PJ#LI6prKRQEye1ewM#-S@=UU<*o1(mzhH;oDX>wI*gCZ}Iun{P93dJ=p>K|I6e zpSb5r7vBJO9bMsX*7!oH?4?`y1|sDtu8;P8W#%>EdZvN#9><$`bN!WqNQ@vVs;CFF&7qH;~>4FF#+9AIdMzczi-) zd_w<;v@vK{QdBHqOI%8H+(;ibBt2nXV|f*c_F<6yg1P9=F^?hOufYoC`4awWRnrt z`_b`2#d$v$7ITnLJL&)h2@FkEy>|WjfSvf(DA}x;@3vA|ko9zrm^ao^j5l#pyxp@? z4crUgK>YW2wa92obE4f*y_3Ue8rryIv0`$b?D%5R)y}}8HqM@T$g2!1Xht;uf;P?x1o%%RN3nDQ?HWq|k1h^JJ0$60@^NuIZ{3 za@ya8pFv%Xw|6v{{y+*hBp_rY`tM`3^Y8NSt0DZa?RSoy57qv!>HpUT{l^dhXb69B zK9~D#dLcdkrzyHfyI7-s(^Jr%-5)~yhukhQFKX;>CPCewnEz26+z?3emjUVd&3P_TD8pY*{{rH_gW3Q9 diff --git a/crates/install-wheel-rs/test/test_install_wheel_rs.py b/crates/install-wheel-rs/test/test_install_wheel_rs.py deleted file mode 100644 index f259890d3..000000000 --- a/crates/install-wheel-rs/test/test_install_wheel_rs.py +++ /dev/null @@ -1,35 +0,0 @@ -import platform -from pathlib import Path -from subprocess import check_call, check_output - - -def check_installed(venv: Path) -> bool: - """true: installed and working, false: not installed, borked: exception""" - try: - output = check_output( - [ - venv.joinpath( - "Scripts" if platform.system() == "Windows" else "bin" - ).joinpath("upsidedown") - ], - input="hello world!", - text=True, - ).strip() - except FileNotFoundError: - return False - assert output == "¡pꞁɹoʍ oꞁꞁǝɥ" - return True - - -def test_install_wheel_rs(pytestconfig, tmp_path): - from install_wheel_rs import LockedVenv - - venv = tmp_path.joinpath("venv_test_install_wheel_rs") - check_call(["virtualenv", venv]) - assert not check_installed(venv) - locked_venv = LockedVenv(venv) - wheel = pytestconfig.rootpath.joinpath( - "install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl" - ) - locked_venv.install_wheel(wheel) - assert check_installed(venv) diff --git a/crates/puffin-platform/Cargo.toml b/crates/platform-host/Cargo.toml similarity index 95% rename from crates/puffin-platform/Cargo.toml rename to crates/platform-host/Cargo.toml index d83b6064e..d30020634 100644 --- a/crates/puffin-platform/Cargo.toml +++ b/crates/platform-host/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "puffin-platform" +name = "platform-host" version = "0.1.0" edition.workspace = true rust-version.workspace = true @@ -10,9 +10,10 @@ authors.workspace = true license.workspace = true [dependencies] +pep440_rs = { path = "../pep440-rs" } + glibc_version = { workspace = true } goblin = { workspace = true } -pep440_rs = { path = "../pep440-rs" } platform-info = { workspace = true } plist = { workspace = true } regex = { workspace = true } diff --git a/crates/puffin-platform/src/lib.rs b/crates/platform-host/src/lib.rs similarity index 99% rename from crates/puffin-platform/src/lib.rs rename to crates/platform-host/src/lib.rs index 6ac23d7a0..cbf199e71 100644 --- a/crates/puffin-platform/src/lib.rs +++ b/crates/platform-host/src/lib.rs @@ -11,8 +11,6 @@ use serde::Deserialize; use thiserror::Error; use tracing::trace; -pub mod tags; - #[derive(Error, Debug)] pub enum PlatformError { #[error(transparent)] @@ -34,8 +32,12 @@ impl Platform { Ok(Self { os, arch }) } - pub fn is_windows(&self) -> bool { - matches!(self.os, Os::Windows) + pub fn os(&self) -> &Os { + &self.os + } + + pub fn arch(&self) -> Arch { + self.arch } } diff --git a/crates/platform-tags/Cargo.toml b/crates/platform-tags/Cargo.toml new file mode 100644 index 000000000..de859c8f4 --- /dev/null +++ b/crates/platform-tags/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "platform-tags" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +platform-host = { path = "../platform-host" } diff --git a/crates/platform-tags/src/lib.rs b/crates/platform-tags/src/lib.rs new file mode 100644 index 000000000..6b22468fc --- /dev/null +++ b/crates/platform-tags/src/lib.rs @@ -0,0 +1,258 @@ +use platform_host::{Arch, Os, Platform, PlatformError}; + +/// A set of compatible tags for a given Python version and platform, in +/// (`python_tag`, `abi_tag`, `platform_tag`) format. +#[derive(Debug)] +pub struct Tags(Vec<(String, String, String)>); + +impl Tags { + /// Returns the compatible tags for the given Python version and platform. + pub fn from_env(platform: &Platform, python_version: (u8, u8)) -> Result { + let platform_tags = compatible_tags(platform)?; + + let mut tags = Vec::with_capacity(5 * platform_tags.len()); + + // 1. This exact c api version + for platform_tag in &platform_tags { + tags.push(( + format!("cp{}{}", python_version.0, python_version.1), + format!( + "cp{}{}{}", + python_version.0, + python_version.1, + // hacky but that's legacy anyways + if python_version.1 <= 7 { "m" } else { "" } + ), + platform_tag.clone(), + )); + tags.push(( + format!("cp{}{}", python_version.0, python_version.1), + "none".to_string(), + platform_tag.clone(), + )); + } + // 2. abi3 and no abi (e.g. executable binary) + // For some reason 3.2 is the minimum python for the cp abi + for minor in 2..=python_version.1 { + for platform_tag in &platform_tags { + tags.push(( + format!("cp{}{}", python_version.0, minor), + "abi3".to_string(), + platform_tag.clone(), + )); + } + } + // 3. no abi (e.g. executable binary) + for minor in 0..=python_version.1 { + for platform_tag in &platform_tags { + tags.push(( + format!("py{}{}", python_version.0, minor), + "none".to_string(), + platform_tag.clone(), + )); + } + } + // 4. major only + for platform_tag in platform_tags { + tags.push(( + format!("py{}", python_version.0), + "none".to_string(), + platform_tag, + )); + } + // 5. no binary + for minor in 0..=python_version.1 { + tags.push(( + format!("py{}{}", python_version.0, minor), + "none".to_string(), + "any".to_string(), + )); + } + tags.push(( + format!("py{}", python_version.0), + "none".to_string(), + "any".to_string(), + )); + tags.sort(); + Ok(Self(tags)) + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +/// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`, +/// `macosx_11_0_arm64`, or `win_amd64`). +/// +/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2) +/// and "any". +/// +/// Bit of a mess, needs to be cleaned up. +fn compatible_tags(platform: &Platform) -> Result, PlatformError> { + let os = platform.os(); + let arch = platform.arch(); + + let platform_tags = match (&os, arch) { + (Os::Manylinux { major, minor }, _) => { + let mut platform_tags = vec![format!("linux_{}", arch)]; + platform_tags.extend( + (arch.get_minimum_manylinux_minor()..=*minor) + .map(|minor| format!("manylinux_{major}_{minor}_{arch}")), + ); + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) { + platform_tags.push(format!("manylinux2010_{arch}")); + } + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) { + platform_tags.push(format!("manylinux2014_{arch}")); + } + if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) { + platform_tags.push(format!("manylinux1_{arch}")); + } + platform_tags + } + (Os::Musllinux { major, minor }, _) => { + let mut platform_tags = vec![format!("linux_{}", arch)]; + // musl 1.1 is the lowest supported version in musllinux + platform_tags + .extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}"))); + platform_tags + } + (Os::Macos { major, minor }, Arch::X86_64) => { + // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 + let mut platform_tags = vec![]; + match major { + 10 => { + // Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version + // number. The major version was always 10. + for minor in (0..=*minor).rev() { + for binary_format in get_mac_binary_formats(*major, minor, arch) { + platform_tags.push(format!("macosx_{major}_{minor}_{binary_format}")); + } + } + } + value if *value >= 11 => { + // Starting with Mac OS 11, each yearly release bumps the major version number. + // The minor versions are now the midyear updates. + for major in (10..=*major).rev() { + for binary_format in get_mac_binary_formats(major, 0, arch) { + platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); + } + } + // The "universal2" binary format can have a macOS version earlier than 11.0 + // when the x86_64 part of the binary supports that version of macOS. + for minor in (4..=16).rev() { + for binary_format in get_mac_binary_formats(10, minor, arch) { + platform_tags + .push(format!("macosx_{}_{}_{}", 10, minor, binary_format)); + } + } + } + _ => { + return Err(PlatformError::OsVersionDetectionError(format!( + "Unsupported macOS version: {major}", + ))); + } + } + platform_tags + } + (Os::Macos { major, .. }, Arch::Aarch64) => { + // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 + let mut platform_tags = vec![]; + // Starting with Mac OS 11, each yearly release bumps the major version number. + // The minor versions are now the midyear updates. + for major in (10..=*major).rev() { + for binary_format in get_mac_binary_formats(major, 0, arch) { + platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); + } + } + // The "universal2" binary format can have a macOS version earlier than 11.0 + // when the x86_64 part of the binary supports that version of macOS. + platform_tags.extend( + (4..=16) + .rev() + .map(|minor| format!("macosx_{}_{}_universal2", 10, minor)), + ); + platform_tags + } + (Os::Windows, Arch::X86) => { + vec!["win32".to_string()] + } + (Os::Windows, Arch::X86_64) => { + vec!["win_amd64".to_string()] + } + (Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()], + ( + Os::FreeBsd { release } + | Os::NetBsd { release } + | Os::OpenBsd { release } + | Os::Dragonfly { release } + | Os::Haiku { release }, + _, + ) => { + let release = release.replace(['.', '-'], "_"); + vec![format!( + "{}_{}_{}", + os.to_string().to_lowercase(), + release, + arch + )] + } + (Os::Illumos { release, arch }, _) => { + // See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730 + if let Some((major, other)) = release.split_once('_') { + let major_ver: u64 = major.parse().map_err(|err| { + PlatformError::OsVersionDetectionError(format!( + "illumos major version is not a number: {err}" + )) + })?; + if major_ver >= 5 { + // SunOS 5 == Solaris 2 + let os = "solaris".to_string(); + let release = format!("{}_{}", major_ver - 3, other); + let arch = format!("{arch}_64bit"); + return Ok(vec![format!("{}_{}_{}", os, release, arch)]); + } + } + + let os = os.to_string().to_lowercase(); + vec![format!("{}_{}_{}", os, release, arch)] + } + _ => { + return Err(PlatformError::OsVersionDetectionError(format!( + "Unsupported operating system and architecture combination: {os} {arch}" + ))); + } + }; + Ok(platform_tags) +} + +/// Determine the appropriate binary formats for a macOS version. +/// Source: +fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec { + let mut formats = vec![match arch { + Arch::Aarch64 => "arm64".to_string(), + _ => arch.to_string(), + }]; + + if matches!(arch, Arch::X86_64) { + if (major, minor) < (10, 4) { + return vec![]; + } + formats.extend([ + "intel".to_string(), + "fat64".to_string(), + "fat32".to_string(), + ]); + } + + if matches!(arch, Arch::X86_64 | Arch::Aarch64) { + formats.push("universal2".to_string()); + } + + if matches!(arch, Arch::X86_64) { + formats.push("universal".to_string()); + } + + formats +} diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index c4752945c..43eb00333 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -4,11 +4,14 @@ version = "0.1.0" edition = "2021" [dependencies] +pep440_rs = { path = "../pep440-rs" } +pep508_rs = { path = "../pep508-rs" } +platform-tags = { path = "../platform-tags" } puffin-client = { path = "../puffin-client" } puffin-installer = { path = "../puffin-installer" } puffin-interpreter = { path = "../puffin-interpreter" } -puffin-platform = { path = "../puffin-platform" } puffin-package = { path = "../puffin-package" } +platform-host = { path = "../platform-host" } puffin-resolver = { path = "../puffin-resolver" } anyhow = { workspace = true } @@ -16,9 +19,6 @@ clap = { workspace = true, features = ["derive"] } colored = { workspace = true } directories = { workspace = true } futures = { workspace = true } -install-wheel-rs = { workspace = true } -pep508_rs = { path = "../pep508-rs" } -pep440_rs = { path = "../pep440-rs" } tracing = { workspace = true } tracing-tree = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs index 4383e0bb3..bf201db02 100644 --- a/crates/puffin-cli/src/commands/compile.rs +++ b/crates/puffin-cli/src/commands/compile.rs @@ -4,10 +4,10 @@ use std::str::FromStr; use anyhow::Result; use tracing::debug; +use platform_host::Platform; +use platform_tags::Tags; use puffin_client::PypiClientBuilder; use puffin_interpreter::PythonExecutable; -use puffin_platform::tags::Tags; -use puffin_platform::Platform; use crate::commands::ExitStatus; @@ -31,7 +31,7 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result) -> Result let markers = python.markers(); // Determine the compatible platform tags. - let tags = Tags::from_env(&platform, python.version())?; + let tags = Tags::from_env(&platform, python.simple_version())?; // Instantiate a client. let client = { diff --git a/crates/puffin-installer/Cargo.toml b/crates/puffin-installer/Cargo.toml index 724fecfca..036fa2f11 100644 --- a/crates/puffin-installer/Cargo.toml +++ b/crates/puffin-installer/Cargo.toml @@ -10,11 +10,12 @@ authors.workspace = true license.workspace = true [dependencies] +install-wheel-rs = { path = "../install-wheel-rs", default-features = false } puffin-client = { path = "../puffin-client" } puffin-interpreter = { path = "../puffin-interpreter" } +wheel-filename = { path = "../wheel-filename" } anyhow = { workspace = true } -install-wheel-rs = { workspace = true } tempfile = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/puffin-installer/src/lib.rs b/crates/puffin-installer/src/lib.rs index dfd786b97..07b45537d 100644 --- a/crates/puffin-installer/src/lib.rs +++ b/crates/puffin-installer/src/lib.rs @@ -2,13 +2,14 @@ use std::path::Path; use std::str::FromStr; use anyhow::Result; -use install_wheel_rs::{install_wheel, InstallLocation}; use tokio::task::JoinSet; use tokio_util::compat::FuturesAsyncReadCompatExt; use url::Url; +use install_wheel_rs::{install_wheel, InstallLocation}; use puffin_client::{File, PypiClient}; use puffin_interpreter::PythonExecutable; +use wheel_filename::WheelFilename; /// Install a set of wheels into a Python virtual environment. pub async fn install( @@ -40,13 +41,14 @@ pub async fn install( let locked_dir = location.acquire_lock()?; for wheel in wheels { let path = tmp_dir.path().join(&wheel.hashes.sha256); - let filename = install_wheel_rs::WheelFilename::from_str(&wheel.filename)?; + let filename = WheelFilename::from_str(&wheel.filename)?; // TODO(charlie): Should this be async? install_wheel( &locked_dir, std::fs::File::open(path)?, - filename, + &filename, + false, false, &[], "", diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index 76ecb7846..1f977e227 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -12,7 +12,7 @@ license.workspace = true [dependencies] pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } -puffin-platform = { path = "../puffin-platform" } +platform-host = { path = "../platform-host" } anyhow = { workspace = true } serde_json = { workspace = true } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index 0c82417fa..2d1147c7a 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -1,10 +1,10 @@ use std::path::{Path, PathBuf}; use anyhow::Result; + use pep440_rs::Version; use pep508_rs::MarkerEnvironment; - -use puffin_platform::Platform; +use platform_host::Platform; use crate::python_platform::PythonPlatform; diff --git a/crates/puffin-interpreter/src/markers.rs b/crates/puffin-interpreter/src/markers.rs index 7650dc845..03c660788 100644 --- a/crates/puffin-interpreter/src/markers.rs +++ b/crates/puffin-interpreter/src/markers.rs @@ -3,6 +3,7 @@ use std::path::Path; use std::process::{Command, Output}; use anyhow::{Context, Result}; + use pep508_rs::MarkerEnvironment; /// Return the resolved [`MarkerEnvironment`] for the given Python executable. diff --git a/crates/puffin-interpreter/src/python_platform.rs b/crates/puffin-interpreter/src/python_platform.rs index e2f6f51e2..774364b0c 100644 --- a/crates/puffin-interpreter/src/python_platform.rs +++ b/crates/puffin-interpreter/src/python_platform.rs @@ -1,7 +1,7 @@ use std::path::Path; use std::path::PathBuf; -use puffin_platform::Platform; +use platform_host::{Os, Platform}; /// A Python-aware wrapper around [`Platform`]. #[derive(Debug, Clone, Eq, PartialEq)] @@ -10,7 +10,7 @@ pub(crate) struct PythonPlatform<'a>(&'a Platform); impl PythonPlatform<'_> { /// Returns the path to the `python` executable inside a virtual environment. pub(crate) fn venv_python(&self, venv_base: impl AsRef) -> PathBuf { - let python = if self.0.is_windows() { + let python = if matches!(self.0.os(), Os::Windows) { "python.exe" } else { "python" @@ -21,7 +21,7 @@ impl PythonPlatform<'_> { /// Returns the directory in which the binaries are stored inside a virtual environment. pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef) -> PathBuf { let venv = venv_base.as_ref(); - if self.0.is_windows() { + if matches!(self.0.os(), Os::Windows) { let bin_dir = venv.join("Scripts"); if bin_dir.join("python.exe").exists() { return bin_dir; diff --git a/crates/puffin-package/Cargo.toml b/crates/puffin-package/Cargo.toml index 68ce5127e..9ea5d01a8 100644 --- a/crates/puffin-package/Cargo.toml +++ b/crates/puffin-package/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] } -puffin-platform = { path = "../puffin-platform" } +platform-host = { path = "../platform-host" } anyhow = { workspace = true } mailparse = { workspace = true } diff --git a/crates/puffin-package/src/lib.rs b/crates/puffin-package/src/lib.rs index ee31b2a2a..e00867e2a 100644 --- a/crates/puffin-package/src/lib.rs +++ b/crates/puffin-package/src/lib.rs @@ -1,4 +1,3 @@ pub mod metadata; pub mod package_name; pub mod requirements; -pub mod wheel; diff --git a/crates/puffin-package/src/metadata.rs b/crates/puffin-package/src/metadata.rs index e7f8a8804..dea24e15d 100644 --- a/crates/puffin-package/src/metadata.rs +++ b/crates/puffin-package/src/metadata.rs @@ -5,11 +5,12 @@ use std::io; use std::str::FromStr; use mailparse::{MailHeaderMap, MailParseError}; -use pep440_rs::{Pep440Error, Version, VersionSpecifiers}; -use pep508_rs::{Pep508Error, Requirement}; use serde::{Deserialize, Serialize}; use thiserror::Error; +use pep440_rs::{Pep440Error, Version, VersionSpecifiers}; +use pep508_rs::{Pep508Error, Requirement}; + /// Python Package Metadata 2.1 as specified in /// /// diff --git a/crates/puffin-package/src/requirements.rs b/crates/puffin-package/src/requirements.rs index 968aa77f3..89089c24f 100644 --- a/crates/puffin-package/src/requirements.rs +++ b/crates/puffin-package/src/requirements.rs @@ -4,6 +4,7 @@ use std::str::FromStr; use anyhow::Result; use memchr::{memchr2, memchr_iter}; + use pep508_rs::{Pep508Error, Requirement}; #[derive(Debug)] diff --git a/crates/puffin-platform/src/tags.rs b/crates/puffin-platform/src/tags.rs deleted file mode 100644 index 84e49ab9f..000000000 --- a/crates/puffin-platform/src/tags.rs +++ /dev/null @@ -1,267 +0,0 @@ -use crate::{Arch, Os, Platform, PlatformError}; - -/// A set of compatible tags for a given Python version and platform, in -/// (`python_tag`, `abi_tag`, `platform_tag`) format. -#[derive(Debug)] -pub struct Tags(Vec<(String, String, String)>); - -impl Tags { - /// Returns the compatible tags for the given Python version and platform. - pub fn from_env( - platform: &Platform, - python_version: &pep440_rs::Version, - ) -> Result { - let python_version = (python_version.release[0], python_version.release[1]); - - let platform_tags = platform.compatible_tags()?; - - let mut tags = Vec::with_capacity(5 * platform_tags.len()); - - // 1. This exact c api version - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - format!( - "cp{}{}{}", - python_version.0, - python_version.1, - // hacky but that's legacy anyways - if python_version.1 <= 7 { "m" } else { "" } - ), - platform_tag.clone(), - )); - tags.push(( - format!("cp{}{}", python_version.0, python_version.1), - "none".to_string(), - platform_tag.clone(), - )); - } - // 2. abi3 and no abi (e.g. executable binary) - // For some reason 3.2 is the minimum python for the cp abi - for minor in 2..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("cp{}{}", python_version.0, minor), - "abi3".to_string(), - platform_tag.clone(), - )); - } - } - // 3. no abi (e.g. executable binary) - for minor in 0..=python_version.1 { - for platform_tag in &platform_tags { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - platform_tag.clone(), - )); - } - } - // 4. major only - for platform_tag in platform_tags { - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - platform_tag, - )); - } - // 5. no binary - for minor in 0..=python_version.1 { - tags.push(( - format!("py{}{}", python_version.0, minor), - "none".to_string(), - "any".to_string(), - )); - } - tags.push(( - format!("py{}", python_version.0), - "none".to_string(), - "any".to_string(), - )); - tags.sort(); - Ok(Self(tags)) - } - - pub fn iter(&self) -> impl Iterator { - self.0.iter() - } -} - -impl Platform { - /// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`, - /// `macosx_11_0_arm64`, or `win_amd64`). - /// - /// We have two cases: Actual platform specific tags (including "merged" tags such as universal2) - /// and "any". - /// - /// Bit of a mess, needs to be cleaned up. - fn compatible_tags(&self) -> Result, PlatformError> { - let os = &self.os; - let arch = self.arch; - - let platform_tags = match (&os, arch) { - (Os::Manylinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - platform_tags.extend( - (arch.get_minimum_manylinux_minor()..=*minor) - .map(|minor| format!("manylinux_{major}_{minor}_{arch}")), - ); - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) { - platform_tags.push(format!("manylinux2010_{arch}")); - } - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) { - platform_tags.push(format!("manylinux2014_{arch}")); - } - if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) { - platform_tags.push(format!("manylinux1_{arch}")); - } - platform_tags - } - (Os::Musllinux { major, minor }, _) => { - let mut platform_tags = vec![format!("linux_{}", arch)]; - // musl 1.1 is the lowest supported version in musllinux - platform_tags - .extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}"))); - platform_tags - } - (Os::Macos { major, minor }, Arch::X86_64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - match major { - 10 => { - // Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version - // number. The major version was always 10. - for minor in (0..=*minor).rev() { - for binary_format in get_mac_binary_formats(*major, minor, arch) { - platform_tags - .push(format!("macosx_{major}_{minor}_{binary_format}")); - } - } - } - value if *value >= 11 => { - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=*major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags - .push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - for minor in (4..=16).rev() { - for binary_format in get_mac_binary_formats(10, minor, arch) { - platform_tags - .push(format!("macosx_{}_{}_{}", 10, minor, binary_format)); - } - } - } - _ => { - return Err(PlatformError::OsVersionDetectionError(format!( - "Unsupported macOS version: {major}", - ))); - } - } - platform_tags - } - (Os::Macos { major, .. }, Arch::Aarch64) => { - // Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346 - let mut platform_tags = vec![]; - // Starting with Mac OS 11, each yearly release bumps the major version number. - // The minor versions are now the midyear updates. - for major in (10..=*major).rev() { - for binary_format in get_mac_binary_formats(major, 0, arch) { - platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format)); - } - } - // The "universal2" binary format can have a macOS version earlier than 11.0 - // when the x86_64 part of the binary supports that version of macOS. - platform_tags.extend( - (4..=16) - .rev() - .map(|minor| format!("macosx_{}_{}_universal2", 10, minor)), - ); - platform_tags - } - (Os::Windows, Arch::X86) => { - vec!["win32".to_string()] - } - (Os::Windows, Arch::X86_64) => { - vec!["win_amd64".to_string()] - } - (Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()], - ( - Os::FreeBsd { release } - | Os::NetBsd { release } - | Os::OpenBsd { release } - | Os::Dragonfly { release } - | Os::Haiku { release }, - _, - ) => { - let release = release.replace(['.', '-'], "_"); - vec![format!( - "{}_{}_{}", - os.to_string().to_lowercase(), - release, - arch - )] - } - (Os::Illumos { release, arch }, _) => { - // See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730 - if let Some((major, other)) = release.split_once('_') { - let major_ver: u64 = major.parse().map_err(|err| { - PlatformError::OsVersionDetectionError(format!( - "illumos major version is not a number: {err}" - )) - })?; - if major_ver >= 5 { - // SunOS 5 == Solaris 2 - let os = "solaris".to_string(); - let release = format!("{}_{}", major_ver - 3, other); - let arch = format!("{arch}_64bit"); - return Ok(vec![format!("{}_{}_{}", os, release, arch)]); - } - } - - let os = os.to_string().to_lowercase(); - vec![format!("{}_{}_{}", os, release, arch)] - } - _ => { - return Err(PlatformError::OsVersionDetectionError(format!( - "Unsupported operating system and architecture combination: {os} {arch}" - ))); - } - }; - Ok(platform_tags) - } -} - -/// Determine the appropriate binary formats for a macOS version. -/// Source: -fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec { - let mut formats = vec![match arch { - Arch::Aarch64 => "arm64".to_string(), - _ => arch.to_string(), - }]; - - if matches!(arch, Arch::X86_64) { - if (major, minor) < (10, 4) { - return vec![]; - } - formats.extend([ - "intel".to_string(), - "fat64".to_string(), - "fat32".to_string(), - ]); - } - - if matches!(arch, Arch::X86_64 | Arch::Aarch64) { - formats.push("universal2".to_string()); - } - - if matches!(arch, Arch::X86_64) { - formats.push("universal".to_string()); - } - - formats -} diff --git a/crates/puffin-resolver/Cargo.toml b/crates/puffin-resolver/Cargo.toml index 30d6bbedd..6b32a833e 100644 --- a/crates/puffin-resolver/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -10,9 +10,11 @@ authors.workspace = true license.workspace = true [dependencies] +platform-tags = { path = "../platform-tags" } puffin-client = { path = "../puffin-client" } -puffin-platform = { path = "../puffin-platform" } puffin-package = { path = "../puffin-package" } +platform-host = { path = "../platform-host" } +wheel-filename = { path = "../wheel-filename" } anyhow = { workspace = true } bitflags = { workspace = true } diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index b56752c59..a00b13d3b 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -5,16 +5,16 @@ use anyhow::Result; use bitflags::bitflags; use futures::future::Either; use futures::{StreamExt, TryFutureExt}; -use pep440_rs::Version; -use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl}; use tracing::debug; +use pep440_rs::Version; +use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl}; +use platform_tags::Tags; use puffin_client::{File, PypiClient, SimpleJson}; use puffin_package::metadata::Metadata21; use puffin_package::package_name::PackageName; use puffin_package::requirements::Requirements; -use puffin_package::wheel::WheelFilename; -use puffin_platform::tags::Tags; +use wheel_filename::WheelFilename; #[derive(Debug)] pub struct Resolution(HashMap); diff --git a/crates/wheel-filename/Cargo.toml b/crates/wheel-filename/Cargo.toml new file mode 100644 index 000000000..d0a99b10f --- /dev/null +++ b/crates/wheel-filename/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "wheel-filename" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +platform-tags = { path = "../platform-tags" } + +thiserror = { workspace = true } \ No newline at end of file diff --git a/crates/puffin-package/src/wheel.rs b/crates/wheel-filename/src/lib.rs similarity index 98% rename from crates/puffin-package/src/wheel.rs rename to crates/wheel-filename/src/lib.rs index 7a2b807f5..acebea2e2 100644 --- a/crates/puffin-package/src/wheel.rs +++ b/crates/wheel-filename/src/lib.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use thiserror::Error; -use puffin_platform::tags::Tags; +use platform_tags::Tags; #[derive(Debug, Clone, Eq, PartialEq)] pub struct WheelFilename {