Use local copy of install-wheel-rs (#34)

This PR modifies the `install-wheel-rs` (and a few other crates) to get
everything playing nicely. Specifically, CI should pass, and all these
crates now use workspace dependencies between one another.

As part of this change, I split out the wheel name parsing into its own
`wheel-filename` crate, and the compatibility tag parsing into its own
`platform-tags` crate.
This commit is contained in:
Charlie Marsh 2023-10-06 21:43:55 -04:00 committed by GitHub
parent e824fe6d2b
commit ae28552b3a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 516 additions and 1567 deletions

71
Cargo.lock generated
View file

@ -1111,20 +1111,23 @@ dependencies = [
[[package]] [[package]]
name = "install-wheel-rs" name = "install-wheel-rs"
version = "0.0.1" version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd8c1fd75d1c686f0e18aa3dd60532f119578b32501517576dd539e0be65f1e"
dependencies = [ dependencies = [
"base64 0.21.4", "clap",
"configparser", "configparser",
"csv", "csv",
"data-encoding",
"fs-err", "fs-err",
"fs2", "fs2",
"glibc_version", "glibc_version",
"goblin", "goblin",
"indoc 2.0.4",
"mailparse", "mailparse",
"once_cell", "once_cell",
"platform-host",
"platform-info", "platform-info",
"plist", "plist",
"pyo3",
"rayon",
"regex", "regex",
"rfc2047-decoder", "rfc2047-decoder",
"serde", "serde",
@ -1134,7 +1137,9 @@ dependencies = [
"tempfile", "tempfile",
"thiserror", "thiserror",
"tracing", "tracing",
"tracing-subscriber",
"walkdir", "walkdir",
"wheel-filename",
"zip", "zip",
] ]
@ -1561,6 +1566,22 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "platform-host"
version = "0.1.0"
dependencies = [
"glibc_version",
"goblin",
"pep440_rs",
"platform-info",
"plist",
"regex",
"serde",
"target-lexicon",
"thiserror",
"tracing",
]
[[package]] [[package]]
name = "platform-info" name = "platform-info"
version = "2.0.2" version = "2.0.2"
@ -1571,6 +1592,13 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "platform-tags"
version = "0.1.0"
dependencies = [
"platform-host",
]
[[package]] [[package]]
name = "plist" name = "plist"
version = "1.5.0" version = "1.5.0"
@ -1646,14 +1674,14 @@ dependencies = [
"colored", "colored",
"directories", "directories",
"futures", "futures",
"install-wheel-rs",
"pep440_rs", "pep440_rs",
"pep508_rs", "pep508_rs",
"platform-host",
"platform-tags",
"puffin-client", "puffin-client",
"puffin-installer", "puffin-installer",
"puffin-interpreter", "puffin-interpreter",
"puffin-package", "puffin-package",
"puffin-platform",
"puffin-resolver", "puffin-resolver",
"tempfile", "tempfile",
"tokio", "tokio",
@ -1693,6 +1721,7 @@ dependencies = [
"tokio-util", "tokio-util",
"tracing", "tracing",
"url", "url",
"wheel-filename",
] ]
[[package]] [[package]]
@ -1702,7 +1731,7 @@ dependencies = [
"anyhow", "anyhow",
"pep440_rs", "pep440_rs",
"pep508_rs", "pep508_rs",
"puffin-platform", "platform-host",
"serde_json", "serde_json",
"tracing", "tracing",
] ]
@ -1719,29 +1748,13 @@ dependencies = [
"once_cell", "once_cell",
"pep440_rs", "pep440_rs",
"pep508_rs", "pep508_rs",
"puffin-platform", "platform-host",
"regex", "regex",
"rfc2047-decoder", "rfc2047-decoder",
"serde", "serde",
"thiserror", "thiserror",
] ]
[[package]]
name = "puffin-platform"
version = "0.1.0"
dependencies = [
"glibc_version",
"goblin",
"pep440_rs",
"platform-info",
"plist",
"regex",
"serde",
"target-lexicon",
"thiserror",
"tracing",
]
[[package]] [[package]]
name = "puffin-resolver" name = "puffin-resolver"
version = "0.1.0" version = "0.1.0"
@ -1751,10 +1764,12 @@ dependencies = [
"futures", "futures",
"pep440_rs", "pep440_rs",
"pep508_rs", "pep508_rs",
"platform-host",
"platform-tags",
"puffin-client", "puffin-client",
"puffin-package", "puffin-package",
"puffin-platform",
"tracing", "tracing",
"wheel-filename",
] ]
[[package]] [[package]]
@ -2878,6 +2893,14 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "wheel-filename"
version = "0.1.0"
dependencies = [
"platform-tags",
"thiserror",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

View file

@ -16,12 +16,16 @@ anyhow = { version = "1.0.75" }
bitflags = { version = "2.4.0" } bitflags = { version = "2.4.0" }
clap = { version = "4.4.6" } clap = { version = "4.4.6" }
colored = { version = "2.0.4" } colored = { version = "2.0.4" }
configparser = { version = "3.0.2" }
csv = { version = "1.3.0" }
data-encoding = { version = "2.4.0" }
directories = { version = "5.0.1" } directories = { version = "5.0.1" }
fs-err = { version = "2.9.0" }
fs2 = { version = "0.4.3" }
futures = { version = "0.3.28" } futures = { version = "0.3.28" }
glibc_version = { version = "0.1.2" } glibc_version = { version = "0.1.2" }
goblin = { version = "0.7.1" } goblin = { version = "0.7.1" }
http-cache-reqwest = { version = "0.11.3" } http-cache-reqwest = { version = "0.11.3" }
install-wheel-rs = { version = "0.0.1" }
mailparse = { version = "0.14.0" } mailparse = { version = "0.14.0" }
memchr = { version = "2.6.4" } memchr = { version = "2.6.4" }
once_cell = { version = "1.18.0" } once_cell = { version = "1.18.0" }
@ -34,6 +38,7 @@ reqwest-retry = { version = "0.3.0" }
rfc2047-decoder = { version = "1.0.1" } rfc2047-decoder = { version = "1.0.1" }
serde = { version = "1.0.188" } serde = { version = "1.0.188" }
serde_json = { version = "1.0.107" } serde_json = { version = "1.0.107" }
sha2 = { version = "0.10.8" }
target-lexicon = { version = "0.12.11" } target-lexicon = { version = "0.12.11" }
tempfile = { version = "3.8.0" } tempfile = { version = "3.8.0" }
thiserror = { version = "1.0.49" } thiserror = { version = "1.0.49" }
@ -44,3 +49,5 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
tracing-tree = { version = "0.2.5" } tracing-tree = { version = "0.2.5" }
unicode-width = { version = "0.1.8" } unicode-width = { version = "0.1.8" }
url = { version = "2.4.1" } url = { version = "2.4.1" }
walkdir = { version = "2.4.0" }
zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

View file

@ -8,6 +8,14 @@ Utilities for interacting with Python version numbers and specifiers.
Utilities for interacting with [PEP 508](https://peps.python.org/pep-0508/) dependency specifiers. Utilities for interacting with [PEP 508](https://peps.python.org/pep-0508/) dependency specifiers.
## [platform-host](./platform-host)
Functionality for detecting the current platform (operating system, architecture, etc.).
## [platform-tags](./platform-tags)
Functionality for parsing and inferring Python platform tags as per [PEP 425](https://peps.python.org/pep-0425/).
## [puffin-cli](./puffin-cli) ## [puffin-cli](./puffin-cli)
Command-line interface for the Puffin package manager. Command-line interface for the Puffin package manager.
@ -28,10 +36,10 @@ Functionality for detecting and leveraging the current Python interpreter.
Types and functionality for working with Python packages, e.g., parsing wheel files. Types and functionality for working with Python packages, e.g., parsing wheel files.
## [puffin-platform](./puffin-platform)
Functionality for detecting the current platform (operating system, architecture, etc.).
## [puffin-resolver](./puffin-resolver) ## [puffin-resolver](./puffin-resolver)
Functionality for resolving Python packages and their dependencies. Functionality for resolving Python packages and their dependencies.
## [wheel-filename](./wheel-filename)
Functionality for parsing wheel filenames as per [PEP 427](https://peps.python.org/pep-0427/).

View file

@ -9,36 +9,37 @@ keywords = ["wheel", "python"]
[lib] [lib]
name = "install_wheel_rs" name = "install_wheel_rs"
# https://github.com/PyO3/maturin/issues/1080 :((
#crate-type = ["cdylib", "rlib"]
[dependencies] [dependencies]
clap = { version = "4.4.6", optional = true, features = ["derive", "env"] } platform-host = { path = "../platform-host" }
configparser = "3.0.2" wheel-filename = { path = "../wheel-filename" }
csv = "1.2.2"
data-encoding = "2.4.0" clap = { workspace = true, optional = true, features = ["derive", "env"] }
configparser = { workspace = true }
csv = { workspace = true }
data-encoding = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
fs2 = { workspace = true } fs2 = { workspace = true }
glibc_version = "0.1.2" glibc_version = { workspace = true }
goblin = "0.7.1" goblin = { workspace = true }
mailparse = "0.14.0" mailparse = { workspace = true }
once_cell = "1.18.0" once_cell = { workspace = true }
platform-info = "2.0.2" platform-info = { workspace = true }
plist = "1.5.0" plist = { workspace = true }
pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true } pyo3 = { version = "0.19.2", features = ["extension-module", "abi3-py37"], optional = true }
rayon = { version = "1.8.0", optional = true } rayon = { version = "1.8.0", optional = true }
regex = { workspace = true } regex = { workspace = true }
rfc2047-decoder = "1.0.1" rfc2047-decoder = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true } serde_json = { workspace = true }
sha2 = { workspace = true } sha2 = { workspace = true }
target-lexicon = "0.12.11" target-lexicon = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
tracing-subscriber = { workspace = true, optional = true } tracing-subscriber = { workspace = true, optional = true }
walkdir = { workspace = true } walkdir = { workspace = true }
zip = { version = "0.6.6", default-features = false, features = ["deflate"] } # no default features for zstd zip = { workspace = true }
[features] [features]
default = ["cli", "parallel"] default = ["cli", "parallel"]
@ -47,7 +48,4 @@ cli = ["clap"]
parallel = ["rayon"] parallel = ["rayon"]
[dev-dependencies] [dev-dependencies]
indoc = { workspace = true } indoc = {version = "2.0.4"}
[package.metadata.dist]
dist = false

View file

@ -1,3 +0,0 @@
class LockedVenv:
def __init__(self, venv: str): ...
def install_wheel(self, wheel: str): ...

View file

@ -1,9 +0,0 @@
[project]
name = "install-wheel-rs"
[tool.maturin]
features = ["python_bindings"]
[build-system]
requires = ["maturin>=1.2,<2.0"]
build-backend = "maturin"

View file

@ -1,12 +0,0 @@
#!/bin/bash
set -e
# cd to project root
cd "$(git rev-parse --show-toplevel)"
rm -f target-maturin/wheels/install_wheel_rs-*.whl
CARGO_TARGET_DIR=target-maturin maturin build --release --strip --no-sdist -m install-wheel-rs/Cargo.toml
.venv/bin/pip uninstall -y install-wheel-rs
.venv/bin/pip install target-maturin/wheels/install_wheel_rs-*.whl
.venv/bin/pytest install-wheel-rs/test
cargo test

View file

@ -139,11 +139,11 @@ impl<T: Deref<Target = Path>> InstallLocation<T> {
.join("site-packages") .join("site-packages")
}; };
site_packages site_packages
.join(format!("{}-{}.dist-info", normalized_name, version)) .join(format!("{normalized_name}-{version}.dist-info"))
.is_dir() .is_dir()
} }
InstallLocation::Monotrail { monotrail_root, .. } => monotrail_root InstallLocation::Monotrail { monotrail_root, .. } => monotrail_root
.join(format!("{}-{}", normalized_name, version)) .join(format!("{normalized_name}-{version}"))
.is_dir(), .is_dir(),
} }
} }

View file

@ -1,37 +1,22 @@
//! Takes a wheel and installs it, either in a venv or for monotrail //! Takes a wheel and installs it, either in a venv or for monotrail.
//!
//! ```no_run use std::io;
//! use std::path::Path;
//! use install_wheel_rs::install_wheel_in_venv;
//!
//! install_wheel_in_venv(
//! "Django-4.2.6-py3-none-any.whl",
//! ".venv",
//! ".venv/bin/python",
//! (3, 8),
//! ).unwrap();
//! ```
use platform_info::PlatformInfoError; use platform_info::PlatformInfoError;
use std::fs::File;
use std::io;
use std::path::Path;
use std::str::FromStr;
use thiserror::Error; use thiserror::Error;
use zip::result::ZipError; use zip::result::ZipError;
pub use install_location::{normalize_name, InstallLocation, LockedDir}; pub use install_location::{normalize_name, InstallLocation, LockedDir};
use platform_host::{Arch, Os};
pub use wheel::{ pub use wheel::{
get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to, get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to,
Script, SHEBANG_PYTHON, Script, SHEBANG_PYTHON,
}; };
pub use wheel_tags::{Arch, CompatibleTags, Os, WheelFilename};
mod install_location; mod install_location;
#[cfg(feature = "python_bindings")] #[cfg(feature = "python_bindings")]
mod python_bindings; mod python_bindings;
mod wheel; mod wheel;
mod wheel_tags;
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error {
@ -50,8 +35,8 @@ pub enum Error {
#[error("The poetry dependency specification (pyproject.toml or poetry.lock) is broken (try `poetry update`?): {0}")] #[error("The poetry dependency specification (pyproject.toml or poetry.lock) is broken (try `poetry update`?): {0}")]
InvalidPoetry(String), InvalidPoetry(String),
/// Doesn't follow file name schema /// Doesn't follow file name schema
#[error("The wheel filename \"{0}\" is invalid: {1}")] #[error(transparent)]
InvalidWheelFileName(String, String), InvalidWheelFileName(#[from] wheel_filename::Error),
#[error("Failed to read the wheel file {0}")] #[error("Failed to read the wheel file {0}")]
Zip(String, #[source] ZipError), Zip(String, #[source] ZipError),
#[error("Failed to run python subcommand")] #[error("Failed to run python subcommand")]
@ -80,44 +65,3 @@ impl Error {
} }
} }
} }
/// High level API: Install a wheel in a virtualenv
///
/// The python interpreter is used for compiling to byte code, the python version for computing
/// the site packages path on unix.
///
/// Returns the tag of the wheel
pub fn install_wheel_in_venv(
wheel: impl AsRef<Path>,
venv: impl AsRef<Path>,
interpreter: impl AsRef<Path>,
major_minor: (u8, u8),
) -> Result<String, Error> {
let venv_base = venv.as_ref().canonicalize()?;
let location = InstallLocation::Venv {
venv_base,
python_version: major_minor,
};
let locked_dir = location.acquire_lock()?;
let filename = wheel
.as_ref()
.file_name()
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy();
let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
install_wheel(
&locked_dir,
File::open(wheel)?,
filename,
false,
true,
&[],
// Only relevant for monotrail style installation
"",
interpreter,
)
}

View file

@ -1,10 +1,11 @@
use clap::Parser; use clap::Parser;
use fs_err::File; use fs_err::File;
use install_wheel_rs::{install_wheel, CompatibleTags, Error, InstallLocation, WheelFilename}; use install_wheel_rs::{install_wheel, Error, InstallLocation};
#[cfg(feature = "rayon")] #[cfg(feature = "rayon")]
use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator};
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use wheel_filename::WheelFilename;
/// Low level install CLI, mainly used for testing /// Low level install CLI, mainly used for testing
#[derive(Parser)] #[derive(Parser)]
@ -45,8 +46,6 @@ fn main() -> Result<(), Error> {
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))? .ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy(); .to_string_lossy();
let filename = WheelFilename::from_str(&filename)?; let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
Ok((wheel, filename)) Ok((wheel, filename))
}) })
.collect::<Result<_, Error>>()?; .collect::<Result<_, Error>>()?;
@ -66,7 +65,7 @@ fn main() -> Result<(), Error> {
install_wheel( install_wheel(
&locked_dir, &locked_dir,
File::open(wheel)?, File::open(wheel)?,
filename, &filename,
args.compile, args.compile,
!args.skip_hashes, !args.skip_hashes,
&[], &[],

View file

@ -1,6 +1,6 @@
#![allow(clippy::format_push_string)] // I will not replace clear and infallible with fallible, io looking code #![allow(clippy::format_push_string)] // I will not replace clear and infallible with fallible, io looking code
use crate::{install_wheel, CompatibleTags, Error, InstallLocation, LockedDir, WheelFilename}; use crate::{install_wheel, Error, InstallLocation, LockedDir};
use pyo3::create_exception; use pyo3::create_exception;
use pyo3::types::PyModule; use pyo3::types::PyModule;
use pyo3::{pyclass, pymethods, pymodule, PyErr, PyResult, Python}; use pyo3::{pyclass, pymethods, pymodule, PyErr, PyResult, Python};
@ -8,6 +8,7 @@ use std::env;
use std::fs::File; use std::fs::File;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::str::FromStr; use std::str::FromStr;
use wheel_filename::WheelFilename;
create_exception!( create_exception!(
install_wheel_rs, install_wheel_rs,
@ -17,11 +18,11 @@ create_exception!(
impl From<Error> for PyErr { impl From<Error> for PyErr {
fn from(err: Error) -> Self { fn from(err: Error) -> Self {
let mut accumulator = format!("Failed to install wheels: {}", err); let mut accumulator = format!("Failed to install wheels: {err}");
let mut current_err: &dyn std::error::Error = &err; let mut current_err: &dyn std::error::Error = &err;
while let Some(cause) = current_err.source() { while let Some(cause) = current_err.source() {
accumulator.push_str(&format!("\n Caused by: {}", cause)); accumulator.push_str(&format!("\n Caused by: {cause}"));
current_err = cause; current_err = cause;
} }
PyWheelInstallerError::new_err(accumulator) PyWheelInstallerError::new_err(accumulator)
@ -36,7 +37,8 @@ struct LockedVenv {
#[pymethods] #[pymethods]
impl LockedVenv { impl LockedVenv {
#[new] #[new]
pub fn new(py: Python, venv: PathBuf) -> PyResult<Self> { #[allow(clippy::needless_pass_by_value)]
pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult<Self> {
Ok(Self { Ok(Self {
location: InstallLocation::Venv { location: InstallLocation::Venv {
venv_base: LockedDir::acquire(&venv)?, venv_base: LockedDir::acquire(&venv)?,
@ -45,7 +47,7 @@ impl LockedVenv {
}) })
} }
pub fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> { pub(crate) fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> {
// Would be nicer through https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath // Would be nicer through https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath
let sys_executable: String = py.import("sys")?.getattr("executable")?.extract()?; let sys_executable: String = py.import("sys")?.getattr("executable")?.extract()?;
@ -56,13 +58,10 @@ impl LockedVenv {
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))? .ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy(); .to_string_lossy();
let filename = WheelFilename::from_str(&filename)?; let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(self.location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
install_wheel( install_wheel(
&self.location, &self.location,
File::open(wheel)?, File::open(wheel)?,
filename, &filename,
true, true,
true, true,
&[], &[],
@ -76,7 +75,7 @@ impl LockedVenv {
} }
#[pymodule] #[pymodule]
pub fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> { pub(crate) fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> {
// Good enough for now // Good enough for now
if env::var_os("RUST_LOG").is_some() { if env::var_os("RUST_LOG").is_some() {
tracing_subscriber::fmt::init(); tracing_subscriber::fmt::init();

View file

@ -1,8 +1,12 @@
#![allow(clippy::needless_borrow)] #![allow(clippy::needless_borrow)]
use crate::install_location::{InstallLocation, LockedDir}; use std::collections::{HashMap, HashSet};
use crate::wheel_tags::WheelFilename; use std::ffi::OsString;
use crate::{normalize_name, Error}; use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
use std::{env, io, iter};
use configparser::ini::Ini; use configparser::ini::Ini;
use data_encoding::BASE64URL_NOPAD; use data_encoding::BASE64URL_NOPAD;
use fs_err as fs; use fs_err as fs;
@ -11,12 +15,6 @@ use mailparse::MailHeaderMap;
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
use std::{env, io, iter};
use tempfile::{tempdir, TempDir}; use tempfile::{tempdir, TempDir};
use tracing::{debug, error, span, warn, Level}; use tracing::{debug, error, span, warn, Level};
use walkdir::WalkDir; use walkdir::WalkDir;
@ -24,12 +22,17 @@ use zip::result::ZipError;
use zip::write::FileOptions; use zip::write::FileOptions;
use zip::{ZipArchive, ZipWriter}; use zip::{ZipArchive, ZipWriter};
use wheel_filename::WheelFilename;
use crate::install_location::{InstallLocation, LockedDir};
use crate::{normalize_name, Error};
/// `#!/usr/bin/env python` /// `#!/usr/bin/env python`
pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python"; pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python";
pub const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe"); pub(crate) const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe");
pub const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe"); pub(crate) const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe");
pub const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe"); pub(crate) const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe");
/// Line in a RECORD file /// Line in a RECORD file
/// <https://www.python.org/dev/peps/pep-0376/#record> /// <https://www.python.org/dev/peps/pep-0376/#record>
@ -43,15 +46,16 @@ pub struct RecordEntry {
pub path: String, pub path: String,
pub hash: Option<String>, pub hash: Option<String>,
#[allow(dead_code)] #[allow(dead_code)]
pub size: Option<usize>, pub size: Option<u64>,
} }
/// Minimal direct_url.json schema /// Minimal `direct_url.json` schema
/// ///
/// <https://packaging.python.org/en/latest/specifications/direct-url/> /// <https://packaging.python.org/en/latest/specifications/direct-url/>
/// <https://www.python.org/dev/peps/pep-0610/> /// <https://www.python.org/dev/peps/pep-0610/>
#[derive(Serialize)] #[derive(Serialize)]
struct DirectUrl { struct DirectUrl {
#[allow(clippy::zero_sized_map_values)]
archive_info: HashMap<(), ()>, archive_info: HashMap<(), ()>,
url: String, url: String,
} }
@ -95,7 +99,7 @@ impl Script {
let captures = script_regex let captures = script_regex
.captures(value) .captures(value)
.ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{}'", value)))?; .ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{value}'")))?;
if let Some(script_extras) = captures.name("extras") { if let Some(script_extras) = captures.name("extras") {
let script_extras = script_extras let script_extras = script_extras
.as_str() .as_str()
@ -130,10 +134,7 @@ from {module} import {import_name}
if __name__ == "__main__": if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
sys.exit({import_name}()) sys.exit({import_name}())
"##, "##
shebang = shebang,
module = module,
import_name = import_name
) )
} }
@ -144,7 +145,7 @@ fn read_scripts_from_section(
extras: Option<&[String]>, extras: Option<&[String]>,
) -> Result<Vec<Script>, Error> { ) -> Result<Vec<Script>, Error> {
let mut scripts = Vec::new(); let mut scripts = Vec::new();
for (script_name, python_location) in scripts_section.iter() { for (script_name, python_location) in scripts_section {
match python_location { match python_location {
Some(value) => { Some(value) => {
if let Some(script) = Script::from_value(script_name, value, extras)? { if let Some(script) = Script::from_value(script_name, value, extras)? {
@ -153,8 +154,7 @@ fn read_scripts_from_section(
} }
None => { None => {
return Err(Error::InvalidWheel(format!( return Err(Error::InvalidWheel(format!(
"[{}] key {} must have a value", "[{section_name}] key {script_name} must have a value"
section_name, script_name
))); )));
} }
} }
@ -162,9 +162,9 @@ fn read_scripts_from_section(
Ok(scripts) Ok(scripts)
} }
/// Parses the entry_points.txt entry in the wheel for console scripts /// Parses the `entry_points.txt` entry in the wheel for console scripts
/// ///
/// Returns (script_name, module, function) /// Returns (`script_name`, module, function)
/// ///
/// Extras are supposed to be ignored, which happens if you pass None for extras /// Extras are supposed to be ignored, which happens if you pass None for extras
fn parse_scripts<R: Read + Seek>( fn parse_scripts<R: Read + Seek>(
@ -177,9 +177,9 @@ fn parse_scripts<R: Read + Seek>(
Ok(mut file) => { Ok(mut file) => {
let mut ini_text = String::new(); let mut ini_text = String::new();
file.read_to_string(&mut ini_text)?; file.read_to_string(&mut ini_text)?;
Ini::new_cs().read(ini_text).map_err(|err| { Ini::new_cs()
Error::InvalidWheel(format!("entry_points.txt is invalid: {}", err)) .read(ini_text)
})? .map_err(|err| Error::InvalidWheel(format!("entry_points.txt is invalid: {err}")))?
} }
Err(ZipError::FileNotFound) => return Ok((Vec::new(), Vec::new())), Err(ZipError::FileNotFound) => return Ok((Vec::new(), Vec::new())),
Err(err) => return Err(Error::from_zip_error(entry_points_path, err)), Err(err) => return Err(Error::from_zip_error(entry_points_path, err)),
@ -204,7 +204,10 @@ fn parse_scripts<R: Read + Seek>(
/// <https://github.com/richo/hashing-copy/blob/d8dd2fdb63c6faf198de0c9e5713d6249cbb5323/src/lib.rs#L10-L52> /// <https://github.com/richo/hashing-copy/blob/d8dd2fdb63c6faf198de0c9e5713d6249cbb5323/src/lib.rs#L10-L52>
/// which in turn got it from std /// which in turn got it from std
/// <https://doc.rust-lang.org/1.58.0/src/std/io/copy.rs.html#128-156> /// <https://doc.rust-lang.org/1.58.0/src/std/io/copy.rs.html#128-156>
pub fn copy_and_hash(reader: &mut impl Read, writer: &mut impl Write) -> io::Result<(u64, String)> { pub(crate) fn copy_and_hash(
reader: &mut impl Read,
writer: &mut impl Write,
) -> io::Result<(u64, String)> {
// TODO: Do we need to support anything besides sha256? // TODO: Do we need to support anything besides sha256?
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
// Same buf size as std. Note that this number is important for performance // Same buf size as std. Note that this number is important for performance
@ -361,7 +364,7 @@ fn get_shebang(location: &InstallLocation<LockedDir>) -> String {
} else { } else {
path path
}; };
format!("#!{}", path) format!("#!{path}")
} else { } else {
// This will use the monotrail binary moonlighting as python. `python` alone doesn't, // This will use the monotrail binary moonlighting as python. `python` alone doesn't,
// we need env to find the python link we put in PATH // we need env to find the python link we put in PATH
@ -369,14 +372,14 @@ fn get_shebang(location: &InstallLocation<LockedDir>) -> String {
} }
} }
/// Ported from https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262
///
/// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual /// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual
/// python after it. /// python after it.
/// ///
/// TODO pyw scripts /// TODO pyw scripts
/// ///
/// TODO: a nice, reproducible-without-distlib rust solution /// TODO: a nice, reproducible-without-distlib rust solution
///
/// <https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262>
fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Error> { fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Error> {
let launcher_bin = match env::consts::ARCH { let launcher_bin = match env::consts::ARCH {
"x84" => LAUNCHER_T32, "x84" => LAUNCHER_T32,
@ -384,9 +387,8 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Erro
"aarch64" => LAUNCHER_T64_ARM, "aarch64" => LAUNCHER_T64_ARM,
arch => { arch => {
let error = format!( let error = format!(
"Don't know how to create windows launchers for script for {}, \ "Don't know how to create windows launchers for script for {arch}, \
only x86, x86_64 and aarch64 (64-bit arm) are supported", only x86, x86_64 and aarch64 (64-bit arm) are supported"
arch
); );
return Err(Error::OsVersionDetection(error)); return Err(Error::OsVersionDetection(error));
} }
@ -414,7 +416,7 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Erro
/// Create the wrapper scripts in the bin folder of the venv for launching console scripts /// Create the wrapper scripts in the bin folder of the venv for launching console scripts
/// ///
/// We also pass venv_base so we can write the same path as pip does /// We also pass `venv_base` so we can write the same path as pip does
/// ///
/// TODO: Test for this launcher directly in install-wheel-rs /// TODO: Test for this launcher directly in install-wheel-rs
fn write_script_entrypoints( fn write_script_entrypoints(
@ -486,16 +488,13 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
// {distribution}-{version}.dist-info/WHEEL is metadata about the archive itself in the same basic key: value format: // {distribution}-{version}.dist-info/WHEEL is metadata about the archive itself in the same basic key: value format:
let data = parse_key_value_file(&mut wheel_text.as_bytes(), "WHEEL")?; let data = parse_key_value_file(&mut wheel_text.as_bytes(), "WHEEL")?;
let wheel_version = if let Some(wheel_version) = let Some(wheel_version) = data.get("Wheel-Version").and_then(|wheel_versions| {
data.get("Wheel-Version").and_then(|wheel_versions| {
if let [wheel_version] = wheel_versions.as_slice() { if let [wheel_version] = wheel_versions.as_slice() {
wheel_version.split_once('.') wheel_version.split_once('.')
} else { } else {
None None
} }
}) { }) else {
wheel_version
} else {
return Err(Error::InvalidWheel( return Err(Error::InvalidWheel(
"Invalid Wheel-Version in WHEEL file".to_string(), "Invalid Wheel-Version in WHEEL file".to_string(),
)); ));
@ -515,7 +514,7 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
))); )));
} }
if wheel_version.1 > "0" { if wheel_version.1 > "0" {
eprint!( warn!(
"Warning: Unsupported wheel minor version (expected {}, got {})", "Warning: Unsupported wheel minor version (expected {}, got {})",
0, wheel_version.1 0, wheel_version.1
); );
@ -553,18 +552,15 @@ fn bytecode_compile(
retries -= 1; retries -= 1;
if status.success() || retries == 0 { if status.success() || retries == 0 {
break (status, lines); break (status, lines);
} else {
warn!(
"Failed to compile {} with python compileall, retrying",
name,
);
} }
warn!("Failed to compile {name} with python compileall, retrying",);
}; };
if !status.success() { if !status.success() {
// lossy because we want the error reporting to survive c̴̞̏ü̸̜̹̈́ŕ̴͉̈ś̷̤ė̵̤͋d̷͙̄ filenames in the zip // lossy because we want the error reporting to survive c̴̞̏ü̸̜̹̈́ŕ̴͉̈ś̷̤ė̵̤͋d̷͙̄ filenames in the zip
return Err(Error::PythonSubcommand(io::Error::new( return Err(Error::PythonSubcommand(io::Error::new(
io::ErrorKind::Other, io::ErrorKind::Other,
format!("Failed to run python compileall, log above: {}", status), format!("Failed to run python compileall, log above: {status}"),
))); )));
} }
@ -606,7 +602,7 @@ fn bytecode_compile(
path: pyc_path.display().to_string(), path: pyc_path.display().to_string(),
hash: None, hash: None,
size: None, size: None,
}) });
} }
Ok(()) Ok(())
@ -655,7 +651,7 @@ fn bytecode_compile_inner(
let line = line.map_err(|err| { let line = line.map_err(|err| {
Error::PythonSubcommand(io::Error::new( Error::PythonSubcommand(io::Error::new(
io::ErrorKind::Other, io::ErrorKind::Other,
format!("Invalid utf-8 returned by python compileall: {}", err), format!("Invalid utf-8 returned by python compileall: {err}"),
)) ))
})?; })?;
lines.push(line); lines.push(line);
@ -671,17 +667,16 @@ fn bytecode_compile_inner(
/// ///
/// lib/python/site-packages/foo/__init__.py and lib/python/site-packages -> foo/__init__.py /// lib/python/site-packages/foo/__init__.py and lib/python/site-packages -> foo/__init__.py
/// lib/marker.txt and lib/python/site-packages -> ../../marker.txt /// lib/marker.txt and lib/python/site-packages -> ../../marker.txt
/// bin/foo_launcher and lib/python/site-packages -> ../../../bin/foo_launcher /// `bin/foo_launcher` and lib/python/site-packages -> ../../../`bin/foo_launcher`
pub fn relative_to(path: &Path, base: &Path) -> Result<PathBuf, Error> { pub fn relative_to(path: &Path, base: &Path) -> Result<PathBuf, Error> {
// Find the longest common prefix, and also return the path stripped from that prefix // Find the longest common prefix, and also return the path stripped from that prefix
let (stripped, common_prefix) = base let (stripped, common_prefix) = base
.ancestors() .ancestors()
.filter_map(|ancestor| { .find_map(|ancestor| {
path.strip_prefix(ancestor) path.strip_prefix(ancestor)
.ok() .ok()
.map(|stripped| (stripped, ancestor)) .map(|stripped| (stripped, ancestor))
}) })
.next()
.ok_or_else(|| { .ok_or_else(|| {
Error::IO(io::Error::new( Error::IO(io::Error::new(
io::ErrorKind::Other, io::ErrorKind::Other,
@ -750,7 +745,7 @@ fn move_folder_recorded(
fn install_script( fn install_script(
site_packages: &Path, site_packages: &Path,
record: &mut [RecordEntry], record: &mut [RecordEntry],
file: DirEntry, file: &DirEntry,
location: &InstallLocation<LockedDir>, location: &InstallLocation<LockedDir>,
) -> Result<(), Error> { ) -> Result<(), Error> {
let path = file.path(); let path = file.path();
@ -818,7 +813,7 @@ fn install_script(
})?; })?;
entry.path = target_path.display().to_string(); entry.path = target_path.display().to_string();
if let Some((size, encoded_hash)) = size_and_encoded_hash { if let Some((size, encoded_hash)) = size_and_encoded_hash {
entry.size = Some(size as usize); entry.size = Some(size);
entry.hash = Some(encoded_hash); entry.hash = Some(encoded_hash);
} }
Ok(()) Ok(())
@ -863,7 +858,7 @@ fn install_data(
continue; continue;
} }
install_script(site_packages, record, file, &location)?; install_script(site_packages, record, &file, &location)?;
} }
} }
Some("headers") => { Some("headers") => {
@ -911,12 +906,12 @@ fn write_file_recorded(
record.push(RecordEntry { record.push(RecordEntry {
path: relative_path.display().to_string(), path: relative_path.display().to_string(),
hash: Some(encoded_hash), hash: Some(encoded_hash),
size: Some(content.as_ref().len()), size: Some(content.as_ref().len() as u64),
}); });
Ok(()) Ok(())
} }
/// Adds INSTALLER, REQUESTED and direct_url.json to the .dist-info dir /// Adds INSTALLER, REQUESTED and `direct_url.json` to the .dist-info dir
fn extra_dist_info( fn extra_dist_info(
site_packages: &Path, site_packages: &Path,
dist_info_prefix: &str, dist_info_prefix: &str,
@ -983,7 +978,7 @@ pub fn parse_key_value_file(
file: &mut impl Read, file: &mut impl Read,
debug_filename: &str, debug_filename: &str,
) -> Result<HashMap<String, Vec<String>>, Error> { ) -> Result<HashMap<String, Vec<String>>, Error> {
let mut data = HashMap::new(); let mut data: HashMap<String, Vec<String>> = HashMap::new();
let file = BufReader::new(file); let file = BufReader::new(file);
for (line_no, line) in file.lines().enumerate() { for (line_no, line) in file.lines().enumerate() {
@ -993,13 +988,12 @@ pub fn parse_key_value_file(
} }
let (key, value) = line.split_once(": ").ok_or_else(|| { let (key, value) = line.split_once(": ").ok_or_else(|| {
Error::InvalidWheel(format!( Error::InvalidWheel(format!(
"Line {} of the {} file is invalid", "Line {line_no} of the {debug_filename} file is invalid"
line_no, debug_filename
)) ))
})?; })?;
data.entry(key.to_string()) data.entry(key.to_string())
.or_insert_with(Vec::new) .or_default()
.push(value.to_string()) .push(value.to_string());
} }
Ok(data) Ok(data)
} }
@ -1011,10 +1005,11 @@ pub fn parse_key_value_file(
/// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl> /// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl>
/// ///
/// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/> /// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/>
#[allow(clippy::too_many_arguments)]
pub fn install_wheel( pub fn install_wheel(
location: &InstallLocation<LockedDir>, location: &InstallLocation<LockedDir>,
reader: impl Read + Seek, reader: impl Read + Seek,
filename: WheelFilename, filename: &WheelFilename,
compile: bool, compile: bool,
check_hashes: bool, check_hashes: bool,
// initially used to the console scripts, currently unused. Keeping it because we likely need // initially used to the console scripts, currently unused. Keeping it because we likely need
@ -1181,11 +1176,11 @@ pub fn install_wheel(
Ok(filename.get_tag()) Ok(filename.get_tag())
} }
/// From https://github.com/PyO3/python-pkginfo-rs
///
/// The metadata name may be uppercase, while the wheel and dist info names are lowercase, or /// The metadata name may be uppercase, while the wheel and dist info names are lowercase, or
/// the metadata name and the dist info name are lowercase, while the wheel name is uppercase. /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase.
/// Either way, we just search the wheel for the name /// Either way, we just search the wheel for the name
///
/// <https://github.com/PyO3/python-pkginfo-rs>
fn find_dist_info( fn find_dist_info(
filename: &WheelFilename, filename: &WheelFilename,
archive: &mut ZipArchive<impl Read + Seek + Sized>, archive: &mut ZipArchive<impl Read + Seek + Sized>,
@ -1205,7 +1200,7 @@ fn find_dist_info(
"Missing .dist-info directory".to_string(), "Missing .dist-info directory".to_string(),
)) ))
} }
[dist_info] => dist_info.to_string(), [dist_info] => (*dist_info).to_string(),
_ => { _ => {
return Err(Error::InvalidWheel(format!( return Err(Error::InvalidWheel(format!(
"Multiple .dist-info directories: {}", "Multiple .dist-info directories: {}",
@ -1216,7 +1211,7 @@ fn find_dist_info(
Ok(dist_info) Ok(dist_info)
} }
/// Adapted from https://github.com/PyO3/python-pkginfo-rs /// <https://github.com/PyO3/python-pkginfo-rs>
fn read_metadata( fn read_metadata(
dist_info_prefix: &str, dist_info_prefix: &str,
archive: &mut ZipArchive<impl Read + Seek + Sized>, archive: &mut ZipArchive<impl Read + Seek + Sized>,
@ -1231,50 +1226,45 @@ fn read_metadata(
let mut mail = b"Content-Type: text/plain; charset=utf-8\n".to_vec(); let mut mail = b"Content-Type: text/plain; charset=utf-8\n".to_vec();
mail.extend_from_slice(&content); mail.extend_from_slice(&content);
let msg = mailparse::parse_mail(&mail) let msg = mailparse::parse_mail(&mail)
.map_err(|err| Error::InvalidWheel(format!("Invalid {}: {}", metadata_file, err)))?; .map_err(|err| Error::InvalidWheel(format!("Invalid {metadata_file}: {err}")))?;
let headers = msg.get_headers(); let headers = msg.get_headers();
let metadata_version = let metadata_version =
headers headers
.get_first_value("Metadata-Version") .get_first_value("Metadata-Version")
.ok_or(Error::InvalidWheel(format!( .ok_or(Error::InvalidWheel(format!(
"No Metadata-Version field in {}", "No Metadata-Version field in {metadata_file}"
metadata_file
)))?; )))?;
// Crude but it should do https://packaging.python.org/en/latest/specifications/core-metadata/#metadata-version // Crude but it should do https://packaging.python.org/en/latest/specifications/core-metadata/#metadata-version
// At time of writing: // At time of writing:
// > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”.
if !(metadata_version.starts_with("1.") || metadata_version.starts_with("2.")) { if !(metadata_version.starts_with("1.") || metadata_version.starts_with("2.")) {
return Err(Error::InvalidWheel(format!( return Err(Error::InvalidWheel(format!(
"Metadata-Version field has unsupported value {}", "Metadata-Version field has unsupported value {metadata_version}"
metadata_version
))); )));
} }
let name = headers let name = headers
.get_first_value("Name") .get_first_value("Name")
.ok_or(Error::InvalidWheel(format!( .ok_or(Error::InvalidWheel(format!(
"No Name field in {}", "No Name field in {metadata_file}"
metadata_file
)))?; )))?;
let version = headers let version = headers
.get_first_value("Version") .get_first_value("Version")
.ok_or(Error::InvalidWheel(format!( .ok_or(Error::InvalidWheel(format!(
"No Version field in {}", "No Version field in {metadata_file}"
metadata_file
)))?; )))?;
Ok((name, version)) Ok((name, version))
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::parse_wheel_version; use std::path::Path;
use crate::wheel::{read_record_file, relative_to};
use crate::{install_wheel, parse_key_value_file, InstallLocation, Script, WheelFilename};
use fs_err as fs;
use indoc::{formatdoc, indoc}; use indoc::{formatdoc, indoc};
use std::fs::File;
use std::path::{Path, PathBuf}; use crate::wheel::{read_record_file, relative_to};
use std::str::FromStr; use crate::{parse_key_value_file, Script};
use tempfile::TempDir;
use super::parse_wheel_version;
#[test] #[test]
fn test_parse_key_value_file() { fn test_parse_key_value_file() {
@ -1331,58 +1321,6 @@ mod test {
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
/// Previously `__pycache__` paths were erroneously absolute
#[test]
fn installed_paths_relative() {
let filename = "colander-0.9.9-py2.py3-none-any.whl";
let wheel = Path::new("../../test-data/wheels").join(filename);
let temp_dir = TempDir::new().unwrap();
// TODO: Would be nicer to pick the default python here, but i don't want to launch a
// subprocess
let python = if cfg!(target_os = "windows") {
PathBuf::from("python.exe")
} else {
PathBuf::from("python3.8")
};
let install_location = InstallLocation::<PathBuf>::Monotrail {
monotrail_root: temp_dir.path().to_path_buf(),
python: python.clone(),
python_version: (3, 8),
}
.acquire_lock()
.unwrap();
install_wheel(
&install_location,
File::open(wheel).unwrap(),
WheelFilename::from_str(&filename).unwrap(),
true,
true,
&[],
"0.9.9",
&python,
)
.unwrap();
let base = temp_dir
.path()
.join("colander")
.join("0.9.9")
.join("py2.py3-none-any");
let mid = if cfg!(windows) {
base.join("Lib")
} else {
base.join("lib").join("python")
};
let record = mid
.join("site-packages")
.join("colander-0.9.9.dist-info")
.join("RECORD");
let record = fs::read_to_string(&record).unwrap();
for line in record.lines() {
assert!(!line.starts_with('/'), "{}", line);
}
}
#[test] #[test]
fn test_relative_to() { fn test_relative_to() {
assert_eq!( assert_eq!(

View file

@ -1,937 +0,0 @@
//! Parses the wheel filename, the current host os/arch and checks wheels for compatibility
use crate::Error;
use fs_err as fs;
use goblin::elf::Elf;
use once_cell::sync::Lazy;
use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI};
use regex::Regex;
use serde::Deserialize;
use std::fmt;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::str::FromStr;
use tracing::trace;
/// The name of a wheel split into its parts ([PEP 491](https://peps.python.org/pep-0491/))
///
/// Ignores the build tag atm.
///
/// ```
/// use std::str::FromStr;
/// use install_wheel_rs::WheelFilename;
///
/// let filename = WheelFilename::from_str("foo-1.0-py32-none-any.whl").unwrap();
/// assert_eq!(filename, WheelFilename {
/// distribution: "foo".to_string(),
/// version: "1.0".to_string(),
/// python_tag: vec!["py32".to_string()],
/// abi_tag: vec!["none".to_string()],
/// platform_tag: vec!["any".to_string()]
/// });
/// let filename = WheelFilename::from_str(
/// "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
/// ).unwrap();
/// assert_eq!(filename, WheelFilename {
/// distribution: "numpy".to_string(),
/// version: "1.26.0".to_string(),
/// python_tag: vec!["cp312".to_string()],
/// abi_tag: vec!["cp312".to_string()],
/// platform_tag: vec![
/// "manylinux_2_17_aarch64".to_string(),
/// "manylinux2014_aarch64".to_string()
/// ]
/// });
/// ```
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WheelFilename {
pub distribution: String,
pub version: String,
pub python_tag: Vec<String>,
pub abi_tag: Vec<String>,
pub platform_tag: Vec<String>,
}
impl FromStr for WheelFilename {
type Err = Error;
fn from_str(filename: &str) -> Result<Self, Self::Err> {
let basename = filename.strip_suffix(".whl").ok_or_else(|| {
Error::InvalidWheelFileName(filename.to_string(), "Must end with .whl".to_string())
})?;
// https://www.python.org/dev/peps/pep-0427/#file-name-convention
match basename.split('-').collect::<Vec<_>>().as_slice() {
// TODO: Build tag precedence
&[distribution, version, _, python_tag, abi_tag, platform_tag]
| &[distribution, version, python_tag, abi_tag, platform_tag] => Ok(WheelFilename {
distribution: distribution.to_string(),
version: version.to_string(),
python_tag: python_tag.split('.').map(String::from).collect(),
abi_tag: abi_tag.split('.').map(String::from).collect(),
platform_tag: platform_tag.split('.').map(String::from).collect(),
}),
_ => Err(Error::InvalidWheelFileName(
filename.to_string(),
"Expected four or five dashes (\"-\") in the filename".to_string(),
)),
}
}
}
impl WheelFilename {
/// Returns Some(precedence) is the wheels are compatible, otherwise none
///
/// Precedence is e.g. used to install newer manylinux wheels over older manylinux wheels
pub fn compatibility(&self, compatible_tags: &CompatibleTags) -> Result<usize, Error> {
compatible_tags
.iter()
.enumerate()
.filter_map(|(precedence, tag)| {
if self.python_tag.contains(&tag.0)
&& self.abi_tag.contains(&tag.1)
&& self.platform_tag.contains(&tag.2)
{
Some(precedence)
} else {
None
}
})
.next()
.ok_or_else(|| Error::IncompatibleWheel {
os: compatible_tags.os.clone(),
arch: compatible_tags.arch,
})
}
/// Effectively undoes the wheel filename parsing step
pub fn get_tag(&self) -> String {
format!(
"{}-{}-{}",
self.python_tag.join("."),
self.abi_tag.join("."),
self.platform_tag.join(".")
)
}
}
/// A platform, defined by the list of compatible wheel tags in order
pub struct CompatibleTags {
pub os: Os,
pub arch: Arch,
pub tags: Vec<(String, String, String)>,
}
impl Deref for CompatibleTags {
type Target = [(String, String, String)];
fn deref(&self) -> &Self::Target {
&self.tags
}
}
/// Returns the compatible tags in a (python_tag, abi_tag, platform_tag) format, ordered from
/// highest precedence to lowest precedence
impl CompatibleTags {
/// Compatible tags for the current operating system and architecture
pub fn current(python_version: (u8, u8)) -> Result<CompatibleTags, Error> {
Self::new(python_version, Os::current()?, Arch::current()?)
}
pub fn new(python_version: (u8, u8), os: Os, arch: Arch) -> Result<CompatibleTags, Error> {
assert_eq!(python_version.0, 3);
let mut tags = Vec::new();
let platform_tags = compatible_platform_tags(&os, &arch)?;
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
Ok(CompatibleTags { os, arch, tags })
}
}
/// All supported operating system
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum Os {
Manylinux { major: u16, minor: u16 },
Musllinux { major: u16, minor: u16 },
Windows,
Macos { major: u16, minor: u16 },
FreeBsd { release: String },
NetBsd { release: String },
OpenBsd { release: String },
Dragonfly { release: String },
Illumos { release: String, arch: String },
Haiku { release: String },
}
impl Os {
fn detect_linux_libc() -> Result<Self, Error> {
let libc = find_libc()?;
let linux = if let Ok(Some((major, minor))) = get_musl_version(&libc) {
Os::Musllinux { major, minor }
} else if let Ok(glibc_ld) = fs::read_link(&libc) {
// Try reading the link first as it's faster
let filename = glibc_ld
.file_name()
.ok_or_else(|| {
Error::OsVersionDetection("Expected the glibc ld to be a file".to_string())
})?
.to_string_lossy();
#[allow(non_upper_case_globals)]
static expr: Lazy<Regex> =
Lazy::new(|| Regex::new(r"ld-(\d{1,3})\.(\d{1,3})\.so").unwrap());
if let Some(capture) = expr.captures(&filename) {
let major = capture.get(1).unwrap().as_str().parse::<u16>().unwrap();
let minor = capture.get(2).unwrap().as_str().parse::<u16>().unwrap();
Os::Manylinux { major, minor }
} else {
trace!("Couldn't use ld filename, using `ldd --version`");
// runs `ldd --version`
let version = glibc_version::get_version().map_err(|err| {
Error::OsVersionDetection(format!(
"Failed to determine glibc version with `ldd --version`: {}",
err
))
})?;
Os::Manylinux {
major: version.major as u16,
minor: version.minor as u16,
}
}
} else {
return Err(Error::OsVersionDetection("Couldn't detect neither glibc version nor musl libc version, at least one of which is required".to_string()));
};
trace!("libc: {}", linux);
Ok(linux)
}
pub fn current() -> Result<Self, Error> {
let target_triple = target_lexicon::HOST;
let os = match target_triple.operating_system {
target_lexicon::OperatingSystem::Linux => Self::detect_linux_libc()?,
target_lexicon::OperatingSystem::Windows => Os::Windows,
target_lexicon::OperatingSystem::MacOSX { major, minor, .. } => {
Os::Macos { major, minor }
}
target_lexicon::OperatingSystem::Darwin => {
let (major, minor) = get_mac_os_version()?;
Os::Macos { major, minor }
}
target_lexicon::OperatingSystem::Netbsd => Os::NetBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Freebsd => Os::FreeBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Openbsd => Os::OpenBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Dragonfly => Os::Dragonfly {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Illumos => {
let platform_info = PlatformInfo::new().map_err(Error::PlatformInfo)?;
Os::Illumos {
release: platform_info.release().to_string_lossy().to_string(),
arch: platform_info.machine().to_string_lossy().to_string(),
}
}
target_lexicon::OperatingSystem::Haiku => Os::Haiku {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
unsupported => {
return Err(Error::OsVersionDetection(format!(
"The operating system {:?} is not supported",
unsupported
)))
}
};
Ok(os)
}
}
impl fmt::Display for Os {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Os::Manylinux { .. } => write!(f, "Manylinux"),
Os::Musllinux { .. } => write!(f, "Musllinux"),
Os::Windows => write!(f, "Windows"),
Os::Macos { .. } => write!(f, "MacOS"),
Os::FreeBsd { .. } => write!(f, "FreeBSD"),
Os::NetBsd { .. } => write!(f, "NetBSD"),
Os::OpenBsd { .. } => write!(f, "OpenBSD"),
Os::Dragonfly { .. } => write!(f, "DragonFly"),
Os::Illumos { .. } => write!(f, "Illumos"),
Os::Haiku { .. } => write!(f, "Haiku"),
}
}
}
/// All supported CPU architectures
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum Arch {
Aarch64,
Armv7L,
Powerpc64Le,
Powerpc64,
X86,
X86_64,
S390X,
}
impl fmt::Display for Arch {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Arch::Aarch64 => write!(f, "aarch64"),
Arch::Armv7L => write!(f, "armv7l"),
Arch::Powerpc64Le => write!(f, "ppc64le"),
Arch::Powerpc64 => write!(f, "ppc64"),
Arch::X86 => write!(f, "i686"),
Arch::X86_64 => write!(f, "x86_64"),
Arch::S390X => write!(f, "s390x"),
}
}
}
impl Arch {
pub fn current() -> Result<Arch, Error> {
let target_triple = target_lexicon::HOST;
let arch = match target_triple.architecture {
target_lexicon::Architecture::X86_64 => Arch::X86_64,
target_lexicon::Architecture::X86_32(_) => Arch::X86,
target_lexicon::Architecture::Arm(_) => Arch::Armv7L,
target_lexicon::Architecture::Aarch64(_) => Arch::Aarch64,
target_lexicon::Architecture::Powerpc64 => Arch::Powerpc64,
target_lexicon::Architecture::Powerpc64le => Arch::Powerpc64Le,
target_lexicon::Architecture::S390x => Arch::S390X,
unsupported => {
return Err(Error::OsVersionDetection(format!(
"The architecture {} is not supported",
unsupported
)));
}
};
Ok(arch)
}
/// Returns the oldest possible Manylinux tag for this architecture
pub fn get_minimum_manylinux_minor(&self) -> u16 {
match self {
// manylinux 2014
Arch::Aarch64 | Arch::Armv7L | Arch::Powerpc64 | Arch::Powerpc64Le | Arch::S390X => 17,
// manylinux 1
Arch::X86 | Arch::X86_64 => 5,
}
}
}
fn get_mac_os_version() -> Result<(u16, u16), Error> {
// This is actually what python does
// https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct SystemVersion {
product_version: String,
}
let system_version: SystemVersion =
plist::from_file("/System/Library/CoreServices/SystemVersion.plist")
.map_err(|err| Error::OsVersionDetection(err.to_string()))?;
let invalid_mac_os_version = || {
Error::OsVersionDetection(format!(
"Invalid mac os version {}",
system_version.product_version
))
};
match system_version
.product_version
.split('.')
.collect::<Vec<&str>>()
.as_slice()
{
[major, minor] | [major, minor, _] => {
let major = major.parse::<u16>().map_err(|_| invalid_mac_os_version())?;
let minor = minor.parse::<u16>().map_err(|_| invalid_mac_os_version())?;
Ok((major, minor))
}
_ => Err(invalid_mac_os_version()),
}
}
/// Determine the appropriate binary formats for a mac os version.
/// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314
fn get_mac_binary_formats(major: u16, minor: u16, arch: &Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}
/// Find musl libc path from executable's ELF header
pub fn find_libc() -> Result<PathBuf, Error> {
let buffer = fs::read("/bin/ls")?;
let error_str = "Couldn't parse /bin/ls for detecting the ld version";
let elf = Elf::parse(&buffer)
.map_err(|err| Error::OsVersionDetection(format!("{}: {}", error_str, err)))?;
if let Some(elf_interpreter) = elf.interpreter {
Ok(PathBuf::from(elf_interpreter))
} else {
Err(Error::OsVersionDetection(error_str.to_string()))
}
}
/// Read the musl version from libc library's output. Taken from maturin
///
/// The libc library should output something like this to stderr::
///
/// musl libc (x86_64)
/// Version 1.2.2
/// Dynamic Program Loader
pub fn get_musl_version(ld_path: impl AsRef<Path>) -> std::io::Result<Option<(u16, u16)>> {
let output = Command::new(ld_path.as_ref())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()?;
let stderr = String::from_utf8_lossy(&output.stderr);
#[allow(non_upper_case_globals)]
static expr: Lazy<Regex> = Lazy::new(|| Regex::new(r"Version (\d{2,4})\.(\d{2,4})").unwrap());
if let Some(capture) = expr.captures(&stderr) {
let major = capture.get(1).unwrap().as_str().parse::<u16>().unwrap();
let minor = capture.get(2).unwrap().as_str().parse::<u16>().unwrap();
return Ok(Some((major, minor)));
}
Ok(None)
}
/// Returns the compatible platform tags from highest precedence to lowest precedence
///
/// Examples: manylinux_2_17, macosx_11_0_arm64, win_amd64
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up. The order also isn't exactly matching that of pip yet,
/// but works good enough in practice
pub fn compatible_platform_tags(os: &Os, arch: &Arch) -> Result<Vec<String>, Error> {
let platform_tags = match (os.clone(), *arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// Use newer manylinux first like pip does
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=minor)
.rev()
.map(|minor| format!("manylinux_{}_{}_{}", major, minor, arch)),
);
if (arch.get_minimum_manylinux_minor()..=minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{}", arch))
}
if (arch.get_minimum_manylinux_minor()..=minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{}", arch))
}
if (arch.get_minimum_manylinux_minor()..=minor).contains(&5) {
platform_tags.push(format!("manylinux1_{}", arch))
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags.extend(
(1..=minor)
.rev()
.map(|minor| format!("musllinux_{}_{}_{}", major, minor, arch)),
);
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=minor).rev() {
for binary_format in get_mac_binary_formats(major, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", major, minor, binary_format));
}
}
}
x if x >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(Error::OsVersionDetection(format!(
"Unsupported mac os version: {}",
major,
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release: _ }
| Os::NetBsd { release: _ }
| Os::OpenBsd { release: _ }
| Os::Dragonfly { release: _ }
| Os::Haiku { release: _ },
_,
) => {
let info = PlatformInfo::new().map_err(Error::PlatformInfo)?;
let release = info.release().to_string_lossy().replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(
Os::Illumos {
mut release,
mut arch,
},
_,
) => {
let mut os = os.to_string().to_lowercase();
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
Error::OsVersionDetection(format!(
"illumos major version is not a number: {}",
err
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
os = "solaris".to_string();
release = format!("{}_{}", major_ver - 3, other);
arch = format!("{}_64bit", arch);
}
}
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(Error::OsVersionDetection(format!(
"Unsupported operating system and architecture combination: {} {}",
os, arch
)));
}
};
Ok(platform_tags)
}
#[cfg(test)]
mod test {
use super::{compatible_platform_tags, WheelFilename};
use crate::{Arch, CompatibleTags, Error, Os};
use fs_err::File;
use std::str::FromStr;
const FILENAMES: &[&str] = &[
"numpy-1.22.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp310-cp310-win_amd64.whl",
"numpy-1.22.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp310-cp310-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp310-cp310-macosx_10_14_x86_64.whl",
"numpy-1.22.2-cp39-cp39-win_amd64.whl",
"numpy-1.22.2-cp39-cp39-win32.whl",
"numpy-1.22.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp39-cp39-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp39-cp39-macosx_10_14_x86_64.whl",
"numpy-1.22.2-cp38-cp38-win_amd64.whl",
"numpy-1.22.2-cp38-cp38-win32.whl",
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl",
"tqdm-4.62.3-py2.py3-none-any.whl",
];
/// Test that we can parse the filenames
#[test]
fn test_wheel_filename_parsing() -> Result<(), Error> {
for filename in FILENAMES {
WheelFilename::from_str(filename)?;
}
Ok(())
}
/// Test that we correctly identify compatible pairs
#[test]
fn test_compatibility() -> Result<(), Error> {
let filenames = [
(
"numpy-1.22.2-cp38-cp38-win_amd64.whl",
((3, 8), Os::Windows, Arch::X86_64),
),
(
"numpy-1.22.2-cp38-cp38-win32.whl",
((3, 8), Os::Windows, Arch::X86),
),
(
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
),
),
(
"numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::Aarch64,
),
),
(
"numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl",
(
(3, 8),
Os::Macos {
major: 11,
minor: 0,
},
Arch::Aarch64,
),
),
(
"numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl",
(
(3, 8),
// Test backwards compatibility here
Os::Macos {
major: 11,
minor: 0,
},
Arch::X86_64,
),
),
(
"ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl",
(
(3, 8),
Os::Macos {
major: 12,
minor: 0,
},
Arch::X86_64,
),
),
(
"ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl",
(
(3, 8),
Os::Macos {
major: 12,
minor: 0,
},
Arch::Aarch64,
),
),
(
"tqdm-4.62.3-py2.py3-none-any.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
),
),
];
for (filename, (python_version, os, arch)) in filenames {
let compatible_tags = CompatibleTags::new(python_version, os, arch)?;
assert!(
WheelFilename::from_str(filename)?
.compatibility(&compatible_tags)
.is_ok(),
"{}",
filename
);
}
Ok(())
}
/// Test that incompatible pairs don't pass is_compatible
#[test]
fn test_compatibility_filter() -> Result<(), Error> {
let compatible_tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?;
let compatible: Vec<&str> = FILENAMES
.iter()
.filter(|filename| {
WheelFilename::from_str(filename)
.unwrap()
.compatibility(&compatible_tags)
.is_ok()
})
.cloned()
.collect();
assert_eq!(
vec![
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"tqdm-4.62.3-py2.py3-none-any.whl"
],
compatible
);
Ok(())
}
fn get_ubuntu_20_04_tags() -> Vec<String> {
serde_json::from_reader(File::open("../../test-data/tags/cp38-ubuntu-20-04.json").unwrap())
.unwrap()
}
/// Check against the tags that packaging.tags reports as compatible
#[test]
fn ubuntu_20_04_compatible() -> Result<(), Error> {
let tags = get_ubuntu_20_04_tags();
for tag in tags {
let compatible_tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?;
assert!(
WheelFilename::from_str(&format!("foo-1.0-{}.whl", tag))?
.compatibility(&compatible_tags)
.is_ok(),
"{}",
tag
)
}
Ok(())
}
/// Check against the tags that packaging.tags reports as compatible
#[test]
fn ubuntu_20_04_list() -> Result<(), Error> {
let expected_tags = get_ubuntu_20_04_tags();
let actual_tags: Vec<String> = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?
.iter()
.map(|(python_tag, abi_tag, platform_tag)| {
format!("{}-{}-{}", python_tag, abi_tag, platform_tag)
})
.collect();
assert_eq!(expected_tags, actual_tags);
Ok(())
}
#[test]
fn test_precedence() {
let tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)
.unwrap();
let pairs = [
(
"greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl"
),
(
"regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"regex-2022.10.31-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl"
),
];
for (higher_str, lower_str) in pairs {
let higher = WheelFilename::from_str(higher_str).unwrap();
let lower = WheelFilename::from_str(lower_str).unwrap();
let higher_precedence = higher.compatibility(&tags).unwrap();
let lower_precedence = lower.compatibility(&tags).unwrap();
assert!(
higher_precedence < lower_precedence,
"{} {} {} {}",
higher_str,
higher_precedence,
lower_str,
lower_precedence
);
}
}
/// Basic does-it-work test
#[test]
fn host_arch() -> Result<(), Error> {
let os = Os::current()?;
let arch = Arch::current()?;
compatible_platform_tags(&os, &arch)?;
Ok(())
}
}

View file

@ -1,35 +0,0 @@
import platform
from pathlib import Path
from subprocess import check_call, check_output
def check_installed(venv: Path) -> bool:
"""true: installed and working, false: not installed, borked: exception"""
try:
output = check_output(
[
venv.joinpath(
"Scripts" if platform.system() == "Windows" else "bin"
).joinpath("upsidedown")
],
input="hello world!",
text=True,
).strip()
except FileNotFoundError:
return False
assert output == "¡pꞁɹoʍ oꞁꞁǝɥ"
return True
def test_install_wheel_rs(pytestconfig, tmp_path):
from install_wheel_rs import LockedVenv
venv = tmp_path.joinpath("venv_test_install_wheel_rs")
check_call(["virtualenv", venv])
assert not check_installed(venv)
locked_venv = LockedVenv(venv)
wheel = pytestconfig.rootpath.joinpath(
"install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl"
)
locked_venv.install_wheel(wheel)
assert check_installed(venv)

View file

@ -1,5 +1,5 @@
[package] [package]
name = "puffin-platform" name = "platform-host"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
rust-version.workspace = true rust-version.workspace = true
@ -10,9 +10,10 @@ authors.workspace = true
license.workspace = true license.workspace = true
[dependencies] [dependencies]
pep440_rs = { path = "../pep440-rs" }
glibc_version = { workspace = true } glibc_version = { workspace = true }
goblin = { workspace = true } goblin = { workspace = true }
pep440_rs = { path = "../pep440-rs" }
platform-info = { workspace = true } platform-info = { workspace = true }
plist = { workspace = true } plist = { workspace = true }
regex = { workspace = true } regex = { workspace = true }

View file

@ -11,8 +11,6 @@ use serde::Deserialize;
use thiserror::Error; use thiserror::Error;
use tracing::trace; use tracing::trace;
pub mod tags;
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum PlatformError { pub enum PlatformError {
#[error(transparent)] #[error(transparent)]
@ -34,8 +32,12 @@ impl Platform {
Ok(Self { os, arch }) Ok(Self { os, arch })
} }
pub fn is_windows(&self) -> bool { pub fn os(&self) -> &Os {
matches!(self.os, Os::Windows) &self.os
}
pub fn arch(&self) -> Arch {
self.arch
} }
} }

View file

@ -0,0 +1,13 @@
[package]
name = "platform-tags"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
platform-host = { path = "../platform-host" }

View file

@ -0,0 +1,258 @@
use platform_host::{Arch, Os, Platform, PlatformError};
/// A set of compatible tags for a given Python version and platform, in
/// (`python_tag`, `abi_tag`, `platform_tag`) format.
#[derive(Debug)]
pub struct Tags(Vec<(String, String, String)>);
impl Tags {
/// Returns the compatible tags for the given Python version and platform.
pub fn from_env(platform: &Platform, python_version: (u8, u8)) -> Result<Self, PlatformError> {
let platform_tags = compatible_tags(platform)?;
let mut tags = Vec::with_capacity(5 * platform_tags.len());
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
tags.sort();
Ok(Self(tags))
}
pub fn iter(&self) -> impl Iterator<Item = &(String, String, String)> {
self.0.iter()
}
}
/// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`,
/// `macosx_11_0_arm64`, or `win_amd64`).
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up.
fn compatible_tags(platform: &Platform) -> Result<Vec<String>, PlatformError> {
let os = platform.os();
let arch = platform.arch();
let platform_tags = match (&os, arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=*minor)
.map(|minor| format!("manylinux_{major}_{minor}_{arch}")),
);
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) {
platform_tags.push(format!("manylinux1_{arch}"));
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags
.extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}")));
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=*minor).rev() {
for binary_format in get_mac_binary_formats(*major, minor, arch) {
platform_tags.push(format!("macosx_{major}_{minor}_{binary_format}"));
}
}
}
value if *value >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported macOS version: {major}",
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release }
| Os::NetBsd { release }
| Os::OpenBsd { release }
| Os::Dragonfly { release }
| Os::Haiku { release },
_,
) => {
let release = release.replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(Os::Illumos { release, arch }, _) => {
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
PlatformError::OsVersionDetectionError(format!(
"illumos major version is not a number: {err}"
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
let os = "solaris".to_string();
let release = format!("{}_{}", major_ver - 3, other);
let arch = format!("{arch}_64bit");
return Ok(vec![format!("{}_{}_{}", os, release, arch)]);
}
}
let os = os.to_string().to_lowercase();
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported operating system and architecture combination: {os} {arch}"
)));
}
};
Ok(platform_tags)
}
/// Determine the appropriate binary formats for a macOS version.
/// Source: <https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314>
fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}

View file

@ -4,11 +4,14 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
platform-tags = { path = "../platform-tags" }
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-installer = { path = "../puffin-installer" } puffin-installer = { path = "../puffin-installer" }
puffin-interpreter = { path = "../puffin-interpreter" } puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" } puffin-package = { path = "../puffin-package" }
platform-host = { path = "../platform-host" }
puffin-resolver = { path = "../puffin-resolver" } puffin-resolver = { path = "../puffin-resolver" }
anyhow = { workspace = true } anyhow = { workspace = true }
@ -16,9 +19,6 @@ clap = { workspace = true, features = ["derive"] }
colored = { workspace = true } colored = { workspace = true }
directories = { workspace = true } directories = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
install-wheel-rs = { workspace = true }
pep508_rs = { path = "../pep508-rs" }
pep440_rs = { path = "../pep440-rs" }
tracing = { workspace = true } tracing = { workspace = true }
tracing-tree = { workspace = true } tracing-tree = { workspace = true }
tracing-subscriber = { workspace = true } tracing-subscriber = { workspace = true }

View file

@ -4,10 +4,10 @@ use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use tracing::debug; use tracing::debug;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_client::PypiClientBuilder; use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags;
use puffin_platform::Platform;
use crate::commands::ExitStatus; use crate::commands::ExitStatus;
@ -31,7 +31,7 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result<ExitStat
let markers = python.markers(); let markers = python.markers();
// Determine the compatible platform tags. // Determine the compatible platform tags.
let tags = Tags::from_env(&platform, python.version())?; let tags = Tags::from_env(&platform, python.simple_version())?;
// Instantiate a client. // Instantiate a client.
let client = { let client = {

View file

@ -4,10 +4,10 @@ use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use tracing::debug; use tracing::debug;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_client::PypiClientBuilder; use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags;
use puffin_platform::Platform;
use crate::commands::ExitStatus; use crate::commands::ExitStatus;
@ -31,7 +31,7 @@ pub(crate) async fn sync(src: &Path, cache: Option<&Path>) -> Result<ExitStatus>
let markers = python.markers(); let markers = python.markers();
// Determine the compatible platform tags. // Determine the compatible platform tags.
let tags = Tags::from_env(&platform, python.version())?; let tags = Tags::from_env(&platform, python.simple_version())?;
// Instantiate a client. // Instantiate a client.
let client = { let client = {

View file

@ -10,11 +10,12 @@ authors.workspace = true
license.workspace = true license.workspace = true
[dependencies] [dependencies]
install-wheel-rs = { path = "../install-wheel-rs", default-features = false }
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" } puffin-interpreter = { path = "../puffin-interpreter" }
wheel-filename = { path = "../wheel-filename" }
anyhow = { workspace = true } anyhow = { workspace = true }
install-wheel-rs = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
url = { workspace = true } url = { workspace = true }

View file

@ -2,13 +2,14 @@ use std::path::Path;
use std::str::FromStr; use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use install_wheel_rs::{install_wheel, InstallLocation};
use tokio::task::JoinSet; use tokio::task::JoinSet;
use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::compat::FuturesAsyncReadCompatExt;
use url::Url; use url::Url;
use install_wheel_rs::{install_wheel, InstallLocation};
use puffin_client::{File, PypiClient}; use puffin_client::{File, PypiClient};
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use wheel_filename::WheelFilename;
/// Install a set of wheels into a Python virtual environment. /// Install a set of wheels into a Python virtual environment.
pub async fn install( pub async fn install(
@ -40,13 +41,14 @@ pub async fn install(
let locked_dir = location.acquire_lock()?; let locked_dir = location.acquire_lock()?;
for wheel in wheels { for wheel in wheels {
let path = tmp_dir.path().join(&wheel.hashes.sha256); let path = tmp_dir.path().join(&wheel.hashes.sha256);
let filename = install_wheel_rs::WheelFilename::from_str(&wheel.filename)?; let filename = WheelFilename::from_str(&wheel.filename)?;
// TODO(charlie): Should this be async? // TODO(charlie): Should this be async?
install_wheel( install_wheel(
&locked_dir, &locked_dir,
std::fs::File::open(path)?, std::fs::File::open(path)?,
filename, &filename,
false,
false, false,
&[], &[],
"", "",

View file

@ -12,7 +12,7 @@ license.workspace = true
[dependencies] [dependencies]
pep440_rs = { path = "../pep440-rs" } pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" } pep508_rs = { path = "../pep508-rs" }
puffin-platform = { path = "../puffin-platform" } platform-host = { path = "../platform-host" }
anyhow = { workspace = true } anyhow = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }

View file

@ -1,10 +1,10 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::Result; use anyhow::Result;
use pep440_rs::Version; use pep440_rs::Version;
use pep508_rs::MarkerEnvironment; use pep508_rs::MarkerEnvironment;
use platform_host::Platform;
use puffin_platform::Platform;
use crate::python_platform::PythonPlatform; use crate::python_platform::PythonPlatform;

View file

@ -3,6 +3,7 @@ use std::path::Path;
use std::process::{Command, Output}; use std::process::{Command, Output};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use pep508_rs::MarkerEnvironment; use pep508_rs::MarkerEnvironment;
/// Return the resolved [`MarkerEnvironment`] for the given Python executable. /// Return the resolved [`MarkerEnvironment`] for the given Python executable.

View file

@ -1,7 +1,7 @@
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use puffin_platform::Platform; use platform_host::{Os, Platform};
/// A Python-aware wrapper around [`Platform`]. /// A Python-aware wrapper around [`Platform`].
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
@ -10,7 +10,7 @@ pub(crate) struct PythonPlatform<'a>(&'a Platform);
impl PythonPlatform<'_> { impl PythonPlatform<'_> {
/// Returns the path to the `python` executable inside a virtual environment. /// Returns the path to the `python` executable inside a virtual environment.
pub(crate) fn venv_python(&self, venv_base: impl AsRef<Path>) -> PathBuf { pub(crate) fn venv_python(&self, venv_base: impl AsRef<Path>) -> PathBuf {
let python = if self.0.is_windows() { let python = if matches!(self.0.os(), Os::Windows) {
"python.exe" "python.exe"
} else { } else {
"python" "python"
@ -21,7 +21,7 @@ impl PythonPlatform<'_> {
/// Returns the directory in which the binaries are stored inside a virtual environment. /// Returns the directory in which the binaries are stored inside a virtual environment.
pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef<Path>) -> PathBuf { pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef<Path>) -> PathBuf {
let venv = venv_base.as_ref(); let venv = venv_base.as_ref();
if self.0.is_windows() { if matches!(self.0.os(), Os::Windows) {
let bin_dir = venv.join("Scripts"); let bin_dir = venv.join("Scripts");
if bin_dir.join("python.exe").exists() { if bin_dir.join("python.exe").exists() {
return bin_dir; return bin_dir;

View file

@ -6,7 +6,7 @@ edition = "2021"
[dependencies] [dependencies]
pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep440_rs = { path = "../pep440-rs", features = ["serde"] }
pep508_rs = { path = "../pep508-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] }
puffin-platform = { path = "../puffin-platform" } platform-host = { path = "../platform-host" }
anyhow = { workspace = true } anyhow = { workspace = true }
mailparse = { workspace = true } mailparse = { workspace = true }

View file

@ -1,4 +1,3 @@
pub mod metadata; pub mod metadata;
pub mod package_name; pub mod package_name;
pub mod requirements; pub mod requirements;
pub mod wheel;

View file

@ -5,11 +5,12 @@ use std::io;
use std::str::FromStr; use std::str::FromStr;
use mailparse::{MailHeaderMap, MailParseError}; use mailparse::{MailHeaderMap, MailParseError};
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
/// Python Package Metadata 2.1 as specified in /// Python Package Metadata 2.1 as specified in
/// <https://packaging.python.org/specifications/core-metadata/> /// <https://packaging.python.org/specifications/core-metadata/>
/// ///

View file

@ -4,6 +4,7 @@ use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use memchr::{memchr2, memchr_iter}; use memchr::{memchr2, memchr_iter};
use pep508_rs::{Pep508Error, Requirement}; use pep508_rs::{Pep508Error, Requirement};
#[derive(Debug)] #[derive(Debug)]

View file

@ -1,267 +0,0 @@
use crate::{Arch, Os, Platform, PlatformError};
/// A set of compatible tags for a given Python version and platform, in
/// (`python_tag`, `abi_tag`, `platform_tag`) format.
#[derive(Debug)]
pub struct Tags(Vec<(String, String, String)>);
impl Tags {
/// Returns the compatible tags for the given Python version and platform.
pub fn from_env(
platform: &Platform,
python_version: &pep440_rs::Version,
) -> Result<Self, PlatformError> {
let python_version = (python_version.release[0], python_version.release[1]);
let platform_tags = platform.compatible_tags()?;
let mut tags = Vec::with_capacity(5 * platform_tags.len());
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
tags.sort();
Ok(Self(tags))
}
pub fn iter(&self) -> impl Iterator<Item = &(String, String, String)> {
self.0.iter()
}
}
impl Platform {
/// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`,
/// `macosx_11_0_arm64`, or `win_amd64`).
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up.
fn compatible_tags(&self) -> Result<Vec<String>, PlatformError> {
let os = &self.os;
let arch = self.arch;
let platform_tags = match (&os, arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=*minor)
.map(|minor| format!("manylinux_{major}_{minor}_{arch}")),
);
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) {
platform_tags.push(format!("manylinux1_{arch}"));
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags
.extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}")));
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=*minor).rev() {
for binary_format in get_mac_binary_formats(*major, minor, arch) {
platform_tags
.push(format!("macosx_{major}_{minor}_{binary_format}"));
}
}
}
value if *value >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported macOS version: {major}",
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release }
| Os::NetBsd { release }
| Os::OpenBsd { release }
| Os::Dragonfly { release }
| Os::Haiku { release },
_,
) => {
let release = release.replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(Os::Illumos { release, arch }, _) => {
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
PlatformError::OsVersionDetectionError(format!(
"illumos major version is not a number: {err}"
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
let os = "solaris".to_string();
let release = format!("{}_{}", major_ver - 3, other);
let arch = format!("{arch}_64bit");
return Ok(vec![format!("{}_{}_{}", os, release, arch)]);
}
}
let os = os.to_string().to_lowercase();
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported operating system and architecture combination: {os} {arch}"
)));
}
};
Ok(platform_tags)
}
}
/// Determine the appropriate binary formats for a macOS version.
/// Source: <https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314>
fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}

View file

@ -10,9 +10,11 @@ authors.workspace = true
license.workspace = true license.workspace = true
[dependencies] [dependencies]
platform-tags = { path = "../platform-tags" }
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" } puffin-package = { path = "../puffin-package" }
platform-host = { path = "../platform-host" }
wheel-filename = { path = "../wheel-filename" }
anyhow = { workspace = true } anyhow = { workspace = true }
bitflags = { workspace = true } bitflags = { workspace = true }

View file

@ -5,16 +5,16 @@ use anyhow::Result;
use bitflags::bitflags; use bitflags::bitflags;
use futures::future::Either; use futures::future::Either;
use futures::{StreamExt, TryFutureExt}; use futures::{StreamExt, TryFutureExt};
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use tracing::debug; use tracing::debug;
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use platform_tags::Tags;
use puffin_client::{File, PypiClient, SimpleJson}; use puffin_client::{File, PypiClient, SimpleJson};
use puffin_package::metadata::Metadata21; use puffin_package::metadata::Metadata21;
use puffin_package::package_name::PackageName; use puffin_package::package_name::PackageName;
use puffin_package::requirements::Requirements; use puffin_package::requirements::Requirements;
use puffin_package::wheel::WheelFilename; use wheel_filename::WheelFilename;
use puffin_platform::tags::Tags;
#[derive(Debug)] #[derive(Debug)]
pub struct Resolution(HashMap<PackageName, PinnedPackage>); pub struct Resolution(HashMap<PackageName, PinnedPackage>);

View file

@ -0,0 +1,15 @@
[package]
name = "wheel-filename"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
platform-tags = { path = "../platform-tags" }
thiserror = { workspace = true }

View file

@ -2,7 +2,7 @@ use std::str::FromStr;
use thiserror::Error; use thiserror::Error;
use puffin_platform::tags::Tags; use platform_tags::Tags;
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
pub struct WheelFilename { pub struct WheelFilename {