mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Use local copy of install-wheel-rs (#34)

Browse source 
This PR modifies the `install-wheel-rs` (and a few other crates) to get
everything playing nicely. Specifically, CI should pass, and all these
crates now use workspace dependencies between one another.

As part of this change, I split out the wheel name parsing into its own
`wheel-filename` crate, and the compatibility tag parsing into its own
`platform-tags` crate.
This commit is contained in:
Charlie Marsh 2023-10-06 21:43:55 -04:00 • committed by GitHub
parent e824fe6d2b
commit ae28552b3a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 516 additions and 1567 deletions
Download patch file Download diff file Expand all files Collapse all files

71
Cargo.lock generated
View file

@ -1111,20 +1111,23 @@ dependencies = [
[[package]]
name = "install-wheel-rs"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd8c1fd75d1c686f0e18aa3dd60532f119578b32501517576dd539e0be65f1e"
dependencies = [
"base64 0.21.4",
"clap",
"configparser",
"csv",
"data-encoding",
"fs-err",
"fs2",
"glibc_version",
"goblin",
"indoc 2.0.4",
"mailparse",
"once_cell",
"platform-host",
"platform-info",
"plist",
"pyo3",
"rayon",
"regex",
"rfc2047-decoder",
"serde",
@ -1134,7 +1137,9 @@ dependencies = [
"tempfile",
"thiserror",
"tracing",
"tracing-subscriber",
"walkdir",
"wheel-filename",
"zip",
]
@ -1561,6 +1566,22 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "platform-host"
version = "0.1.0"
dependencies = [
"glibc_version",
"goblin",
"pep440_rs",
"platform-info",
"plist",
"regex",
"serde",
"target-lexicon",
"thiserror",
"tracing",
]
[[package]]
name = "platform-info"
version = "2.0.2"
@ -1571,6 +1592,13 @@ dependencies = [
"winapi",
]
[[package]]
name = "platform-tags"
version = "0.1.0"
dependencies = [
"platform-host",
]
[[package]]
name = "plist"
version = "1.5.0"
@ -1646,14 +1674,14 @@ dependencies = [
"colored",
"directories",
"futures",
"install-wheel-rs",
"pep440_rs",
"pep508_rs",
"platform-host",
"platform-tags",
"puffin-client",
"puffin-installer",
"puffin-interpreter",
"puffin-package",
"puffin-platform",
"puffin-resolver",
"tempfile",
"tokio",
@ -1693,6 +1721,7 @@ dependencies = [
"tokio-util",
"tracing",
"url",
"wheel-filename",
]
[[package]]
@ -1702,7 +1731,7 @@ dependencies = [
"anyhow",
"pep440_rs",
"pep508_rs",
"puffin-platform",
"platform-host",
"serde_json",
"tracing",
]
@ -1719,29 +1748,13 @@ dependencies = [
"once_cell",
"pep440_rs",
"pep508_rs",
"puffin-platform",
"platform-host",
"regex",
"rfc2047-decoder",
"serde",
"thiserror",
]
[[package]]
name = "puffin-platform"
version = "0.1.0"
dependencies = [
"glibc_version",
"goblin",
"pep440_rs",
"platform-info",
"plist",
"regex",
"serde",
"target-lexicon",
"thiserror",
"tracing",
]
[[package]]
name = "puffin-resolver"
version = "0.1.0"
@ -1751,10 +1764,12 @@ dependencies = [
"futures",
"pep440_rs",
"pep508_rs",
"platform-host",
"platform-tags",
"puffin-client",
"puffin-package",
"puffin-platform",
"tracing",
"wheel-filename",
]
[[package]]
@ -2878,6 +2893,14 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "wheel-filename"
version = "0.1.0"
dependencies = [
"platform-tags",
"thiserror",
]
[[package]]
name = "winapi"
version = "0.3.9"

9
Cargo.toml
View file

@ -16,12 +16,16 @@ anyhow = { version = "1.0.75" }
bitflags = { version = "2.4.0" }
clap = { version = "4.4.6" }
colored = { version = "2.0.4" }
configparser = { version = "3.0.2" }
csv = { version = "1.3.0" }
data-encoding = { version = "2.4.0" }
directories = { version = "5.0.1" }
fs-err = { version = "2.9.0" }
fs2 = { version = "0.4.3" }
futures = { version = "0.3.28" }
glibc_version = { version = "0.1.2" }
goblin = { version = "0.7.1" }
http-cache-reqwest = { version = "0.11.3" }
install-wheel-rs = { version = "0.0.1" }
mailparse = { version = "0.14.0" }
memchr = { version = "2.6.4" }
once_cell = { version = "1.18.0" }
@ -34,6 +38,7 @@ reqwest-retry = { version = "0.3.0" }
rfc2047-decoder = { version = "1.0.1" }
serde = { version = "1.0.188" }
serde_json = { version = "1.0.107" }
sha2 = { version = "0.10.8" }
target-lexicon = { version = "0.12.11" }
tempfile = { version = "3.8.0" }
thiserror = { version = "1.0.49" }
@ -44,3 +49,5 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
tracing-tree = { version = "0.2.5" }
unicode-width = { version = "0.1.8" }
url = { version = "2.4.1" }
walkdir = { version = "2.4.0" }
zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

16
crates/README.md
View file

@ -8,6 +8,14 @@ Utilities for interacting with Python version numbers and specifiers.
Utilities for interacting with [PEP 508](https://peps.python.org/pep-0508/) dependency specifiers.
## [platform-host](./platform-host)
Functionality for detecting the current platform (operating system, architecture, etc.).
## [platform-tags](./platform-tags)
Functionality for parsing and inferring Python platform tags as per [PEP 425](https://peps.python.org/pep-0425/).
## [puffin-cli](./puffin-cli)
Command-line interface for the Puffin package manager.
@ -28,10 +36,10 @@ Functionality for detecting and leveraging the current Python interpreter.
Types and functionality for working with Python packages, e.g., parsing wheel files.
## [puffin-platform](./puffin-platform)
Functionality for detecting the current platform (operating system, architecture, etc.).
## [puffin-resolver](./puffin-resolver)
Functionality for resolving Python packages and their dependencies.
## [wheel-filename](./wheel-filename)
Functionality for parsing wheel filenames as per [PEP 427](https://peps.python.org/pep-0427/).

38
crates/install-wheel-rs/Cargo.toml
View file

@ -9,36 +9,37 @@ keywords = ["wheel", "python"]
[lib]
name = "install_wheel_rs"
# https://github.com/PyO3/maturin/issues/1080 :((
#crate-type = ["cdylib", "rlib"]
[dependencies]
clap = { version = "4.4.6", optional = true, features = ["derive", "env"] }
configparser = "3.0.2"
csv = "1.2.2"
data-encoding = "2.4.0"
platform-host = { path = "../platform-host" }
wheel-filename = { path = "../wheel-filename" }
clap = { workspace = true, optional = true, features = ["derive", "env"] }
configparser = { workspace = true }
csv = { workspace = true }
data-encoding = { workspace = true }
fs-err = { workspace = true }
fs2 = { workspace = true }
glibc_version = "0.1.2"
goblin = "0.7.1"
mailparse = "0.14.0"
once_cell = "1.18.0"
platform-info = "2.0.2"
plist = "1.5.0"
pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true }
glibc_version = { workspace = true }
goblin = { workspace = true }
mailparse = { workspace = true }
once_cell = { workspace = true }
platform-info = { workspace = true }
plist = { workspace = true }
pyo3 = { version = "0.19.2", features = ["extension-module", "abi3-py37"], optional = true }
rayon = { version = "1.8.0", optional = true }
regex = { workspace = true }
rfc2047-decoder = "1.0.1"
rfc2047-decoder = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha2 = { workspace = true }
target-lexicon = "0.12.11"
target-lexicon = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, optional = true }
walkdir = { workspace = true }
zip = { version = "0.6.6", default-features = false, features = ["deflate"] } # no default features for zstd
zip = { workspace = true }
[features]
default = ["cli", "parallel"]
@ -47,7 +48,4 @@ cli = ["clap"]
parallel = ["rayon"]
[dev-dependencies]
indoc = { workspace = true }
[package.metadata.dist]
dist = false
indoc = {version = "2.0.4"}

3
crates/install-wheel-rs/install_wheel_rs.pyi
View file

@ -1,3 +0,0 @@
class LockedVenv:
def __init__(self, venv: str): ...
def install_wheel(self, wheel: str): ...

9
crates/install-wheel-rs/pyproject.toml
View file

@ -1,9 +0,0 @@
[project]
name = "install-wheel-rs"
[tool.maturin]
features = ["python_bindings"]
[build-system]
requires = ["maturin>=1.2,<2.0"]
build-backend = "maturin"

12
crates/install-wheel-rs/run_tests.sh
View file

@ -1,12 +0,0 @@
#!/bin/bash
set -e
# cd to project root
cd "$(git rev-parse --show-toplevel)"
rm -f target-maturin/wheels/install_wheel_rs-*.whl
CARGO_TARGET_DIR=target-maturin maturin build --release --strip --no-sdist -m install-wheel-rs/Cargo.toml
.venv/bin/pip uninstall -y install-wheel-rs
.venv/bin/pip install target-maturin/wheels/install_wheel_rs-*.whl
.venv/bin/pytest install-wheel-rs/test
cargo test

4
crates/install-wheel-rs/src/install_location.rs
View file

@ -139,11 +139,11 @@ impl<T: Deref<Target = Path>> InstallLocation<T> {
.join("site-packages")
};
site_packages
.join(format!("{}-{}.dist-info", normalized_name, version))
.join(format!("{normalized_name}-{version}.dist-info"))
.is_dir()
}
InstallLocation::Monotrail { monotrail_root, .. } => monotrail_root
.join(format!("{}-{}", normalized_name, version))
.join(format!("{normalized_name}-{version}"))
.is_dir(),
}
}

68
crates/install-wheel-rs/src/lib.rs
View file

@ -1,37 +1,22 @@
//! Takes a wheel and installs it, either in a venv or for monotrail
//!
//! ```no_run
//! use std::path::Path;
//! use install_wheel_rs::install_wheel_in_venv;
//!
//! install_wheel_in_venv(
//! "Django-4.2.6-py3-none-any.whl",
//! ".venv",
//! ".venv/bin/python",
//! (3, 8),
//! ).unwrap();
//! ```
//! Takes a wheel and installs it, either in a venv or for monotrail.
use std::io;
use platform_info::PlatformInfoError;
use std::fs::File;
use std::io;
use std::path::Path;
use std::str::FromStr;
use thiserror::Error;
use zip::result::ZipError;
pub use install_location::{normalize_name, InstallLocation, LockedDir};
use platform_host::{Arch, Os};
pub use wheel::{
get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to,
Script, SHEBANG_PYTHON,
};
pub use wheel_tags::{Arch, CompatibleTags, Os, WheelFilename};
mod install_location;
#[cfg(feature = "python_bindings")]
mod python_bindings;
mod wheel;
mod wheel_tags;
#[derive(Error, Debug)]
pub enum Error {
@ -50,8 +35,8 @@ pub enum Error {
#[error("The poetry dependency specification (pyproject.toml or poetry.lock) is broken (try `poetry update`?): {0}")]
InvalidPoetry(String),
/// Doesn't follow file name schema
#[error("The wheel filename \"{0}\" is invalid: {1}")]
InvalidWheelFileName(String, String),
#[error(transparent)]
InvalidWheelFileName(#[from] wheel_filename::Error),
#[error("Failed to read the wheel file {0}")]
Zip(String, #[source] ZipError),
#[error("Failed to run python subcommand")]
@ -80,44 +65,3 @@ impl Error {
}
}
}
/// High level API: Install a wheel in a virtualenv
///
/// The python interpreter is used for compiling to byte code, the python version for computing
/// the site packages path on unix.
///
/// Returns the tag of the wheel
pub fn install_wheel_in_venv(
wheel: impl AsRef<Path>,
venv: impl AsRef<Path>,
interpreter: impl AsRef<Path>,
major_minor: (u8, u8),
) -> Result<String, Error> {
let venv_base = venv.as_ref().canonicalize()?;
let location = InstallLocation::Venv {
venv_base,
python_version: major_minor,
};
let locked_dir = location.acquire_lock()?;
let filename = wheel
.as_ref()
.file_name()
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy();
let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
install_wheel(
&locked_dir,
File::open(wheel)?,
filename,
false,
true,
&[],
// Only relevant for monotrail style installation
"",
interpreter,
)
}

7
crates/install-wheel-rs/src/main.rs
View file

@ -1,10 +1,11 @@
use clap::Parser;
use fs_err::File;
use install_wheel_rs::{install_wheel, CompatibleTags, Error, InstallLocation, WheelFilename};
use install_wheel_rs::{install_wheel, Error, InstallLocation};
#[cfg(feature = "rayon")]
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use std::path::PathBuf;
use std::str::FromStr;
use wheel_filename::WheelFilename;
/// Low level install CLI, mainly used for testing
#[derive(Parser)]
@ -45,8 +46,6 @@ fn main() -> Result<(), Error> {
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy();
let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
Ok((wheel, filename))
})
.collect::<Result<_, Error>>()?;
@ -66,7 +65,7 @@ fn main() -> Result<(), Error> {
install_wheel(
&locked_dir,
File::open(wheel)?,
filename,
&filename,
args.compile,
!args.skip_hashes,
&[],

19
crates/install-wheel-rs/src/python_bindings.rs
View file

@ -1,6 +1,6 @@
#![allow(clippy::format_push_string)] // I will not replace clear and infallible with fallible, io looking code
use crate::{install_wheel, CompatibleTags, Error, InstallLocation, LockedDir, WheelFilename};
use crate::{install_wheel, Error, InstallLocation, LockedDir};
use pyo3::create_exception;
use pyo3::types::PyModule;
use pyo3::{pyclass, pymethods, pymodule, PyErr, PyResult, Python};
@ -8,6 +8,7 @@ use std::env;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use wheel_filename::WheelFilename;
create_exception!(
install_wheel_rs,
@ -17,11 +18,11 @@ create_exception!(
impl From<Error> for PyErr {
fn from(err: Error) -> Self {
let mut accumulator = format!("Failed to install wheels: {}", err);
let mut accumulator = format!("Failed to install wheels: {err}");
let mut current_err: &dyn std::error::Error = &err;
while let Some(cause) = current_err.source() {
accumulator.push_str(&format!("\n Caused by: {}", cause));
accumulator.push_str(&format!("\n Caused by: {cause}"));
current_err = cause;
}
PyWheelInstallerError::new_err(accumulator)
@ -36,7 +37,8 @@ struct LockedVenv {
#[pymethods]
impl LockedVenv {
#[new]
pub fn new(py: Python, venv: PathBuf) -> PyResult<Self> {
#[allow(clippy::needless_pass_by_value)]
pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult<Self> {
Ok(Self {
location: InstallLocation::Venv {
venv_base: LockedDir::acquire(&venv)?,
@ -45,7 +47,7 @@ impl LockedVenv {
})
}
pub fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> {
pub(crate) fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> {
// Would be nicer through https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath
let sys_executable: String = py.import("sys")?.getattr("executable")?.extract()?;
@ -56,13 +58,10 @@ impl LockedVenv {
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
.to_string_lossy();
let filename = WheelFilename::from_str(&filename)?;
let compatible_tags = CompatibleTags::current(self.location.get_python_version())?;
filename.compatibility(&compatible_tags)?;
install_wheel(
&self.location,
File::open(wheel)?,
filename,
&filename,
true,
true,
&[],
@ -76,7 +75,7 @@ impl LockedVenv {
}
#[pymodule]
pub fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> {
pub(crate) fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> {
// Good enough for now
if env::var_os("RUST_LOG").is_some() {
tracing_subscriber::fmt::init();

210
crates/install-wheel-rs/src/wheel.rs
View file

@ -1,8 +1,12 @@
#![allow(clippy::needless_borrow)]
use crate::install_location::{InstallLocation, LockedDir};
use crate::wheel_tags::WheelFilename;
use crate::{normalize_name, Error};
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
use std::{env, io, iter};
use configparser::ini::Ini;
use data_encoding::BASE64URL_NOPAD;
use fs_err as fs;
@ -11,12 +15,6 @@ use mailparse::MailHeaderMap;
use regex::Regex;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
use std::{env, io, iter};
use tempfile::{tempdir, TempDir};
use tracing::{debug, error, span, warn, Level};
use walkdir::WalkDir;
@ -24,12 +22,17 @@ use zip::result::ZipError;
use zip::write::FileOptions;
use zip::{ZipArchive, ZipWriter};
use wheel_filename::WheelFilename;
use crate::install_location::{InstallLocation, LockedDir};
use crate::{normalize_name, Error};
/// `#!/usr/bin/env python`
pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python";
pub const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe");
pub const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe");
pub const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe");
pub(crate) const LAUNCHER_T32: &[u8] = include_bytes!("../windows-launcher/t32.exe");
pub(crate) const LAUNCHER_T64: &[u8] = include_bytes!("../windows-launcher/t64.exe");
pub(crate) const LAUNCHER_T64_ARM: &[u8] = include_bytes!("../windows-launcher/t64-arm.exe");
/// Line in a RECORD file
/// <https://www.python.org/dev/peps/pep-0376/#record>
@ -43,15 +46,16 @@ pub struct RecordEntry {
pub path: String,
pub hash: Option<String>,
#[allow(dead_code)]
pub size: Option<usize>,
pub size: Option<u64>,
}
/// Minimal direct_url.json schema
/// Minimal `direct_url.json` schema
///
/// <https://packaging.python.org/en/latest/specifications/direct-url/>
/// <https://www.python.org/dev/peps/pep-0610/>
#[derive(Serialize)]
struct DirectUrl {
#[allow(clippy::zero_sized_map_values)]
archive_info: HashMap<(), ()>,
url: String,
}
@ -95,7 +99,7 @@ impl Script {
let captures = script_regex
.captures(value)
.ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{}'", value)))?;
.ok_or_else(|| Error::InvalidWheel(format!("invalid console script: '{value}'")))?;
if let Some(script_extras) = captures.name("extras") {
let script_extras = script_extras
.as_str()
@ -130,10 +134,7 @@ from {module} import {import_name}
if __name__ == "__main__":
sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
sys.exit({import_name}())
"##,
shebang = shebang,
module = module,
import_name = import_name
"##
)
}
@ -144,7 +145,7 @@ fn read_scripts_from_section(
extras: Option<&[String]>,
) -> Result<Vec<Script>, Error> {
let mut scripts = Vec::new();
for (script_name, python_location) in scripts_section.iter() {
for (script_name, python_location) in scripts_section {
match python_location {
Some(value) => {
if let Some(script) = Script::from_value(script_name, value, extras)? {
@ -153,8 +154,7 @@ fn read_scripts_from_section(
}
None => {
return Err(Error::InvalidWheel(format!(
"[{}] key {} must have a value",
section_name, script_name
"[{section_name}] key {script_name} must have a value"
)));
}
}
@ -162,9 +162,9 @@ fn read_scripts_from_section(
Ok(scripts)
}
/// Parses the entry_points.txt entry in the wheel for console scripts
/// Parses the `entry_points.txt` entry in the wheel for console scripts
///
/// Returns (script_name, module, function)
/// Returns (`script_name`, module, function)
///
/// Extras are supposed to be ignored, which happens if you pass None for extras
fn parse_scripts<R: Read + Seek>(
@ -177,9 +177,9 @@ fn parse_scripts<R: Read + Seek>(
Ok(mut file) => {
let mut ini_text = String::new();
file.read_to_string(&mut ini_text)?;
Ini::new_cs().read(ini_text).map_err(|err| {
Error::InvalidWheel(format!("entry_points.txt is invalid: {}", err))
})?
Ini::new_cs()
.read(ini_text)
.map_err(|err| Error::InvalidWheel(format!("entry_points.txt is invalid: {err}")))?
}
Err(ZipError::FileNotFound) => return Ok((Vec::new(), Vec::new())),
Err(err) => return Err(Error::from_zip_error(entry_points_path, err)),
@ -204,7 +204,10 @@ fn parse_scripts<R: Read + Seek>(
/// <https://github.com/richo/hashing-copy/blob/d8dd2fdb63c6faf198de0c9e5713d6249cbb5323/src/lib.rs#L10-L52>
/// which in turn got it from std
/// <https://doc.rust-lang.org/1.58.0/src/std/io/copy.rs.html#128-156>
pub fn copy_and_hash(reader: &mut impl Read, writer: &mut impl Write) -> io::Result<(u64, String)> {
pub(crate) fn copy_and_hash(
reader: &mut impl Read,
writer: &mut impl Write,
) -> io::Result<(u64, String)> {
// TODO: Do we need to support anything besides sha256?
let mut hasher = Sha256::new();
// Same buf size as std. Note that this number is important for performance
@ -361,7 +364,7 @@ fn get_shebang(location: &InstallLocation<LockedDir>) -> String {
} else {
path
};
format!("#!{}", path)
format!("#!{path}")
} else {
// This will use the monotrail binary moonlighting as python. `python` alone doesn't,
// we need env to find the python link we put in PATH
@ -369,14 +372,14 @@ fn get_shebang(location: &InstallLocation<LockedDir>) -> String {
}
}
/// Ported from https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262
///
/// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual
/// python after it.
///
/// TODO pyw scripts
///
/// TODO: a nice, reproducible-without-distlib rust solution
///
/// <https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262>
fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Error> {
let launcher_bin = match env::consts::ARCH {
"x84" => LAUNCHER_T32,
@ -384,9 +387,8 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Erro
"aarch64" => LAUNCHER_T64_ARM,
arch => {
let error = format!(
"Don't know how to create windows launchers for script for {}, \
only x86, x86_64 and aarch64 (64-bit arm) are supported",
arch
"Don't know how to create windows launchers for script for {arch}, \
only x86, x86_64 and aarch64 (64-bit arm) are supported"
);
return Err(Error::OsVersionDetection(error));
}
@ -414,7 +416,7 @@ fn windows_script_launcher(launcher_python_script: &str) -> Result<Vec<u8>, Erro
/// Create the wrapper scripts in the bin folder of the venv for launching console scripts
///
/// We also pass venv_base so we can write the same path as pip does
/// We also pass `venv_base` so we can write the same path as pip does
///
/// TODO: Test for this launcher directly in install-wheel-rs
fn write_script_entrypoints(
@ -486,16 +488,13 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
// {distribution}-{version}.dist-info/WHEEL is metadata about the archive itself in the same basic key: value format:
let data = parse_key_value_file(&mut wheel_text.as_bytes(), "WHEEL")?;
let wheel_version = if let Some(wheel_version) =
data.get("Wheel-Version").and_then(|wheel_versions| {
let Some(wheel_version) = data.get("Wheel-Version").and_then(|wheel_versions| {
if let [wheel_version] = wheel_versions.as_slice() {
wheel_version.split_once('.')
} else {
None
}
}) {
wheel_version
} else {
}) else {
return Err(Error::InvalidWheel(
"Invalid Wheel-Version in WHEEL file".to_string(),
));
@ -515,7 +514,7 @@ fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
)));
}
if wheel_version.1 > "0" {
eprint!(
warn!(
"Warning: Unsupported wheel minor version (expected {}, got {})",
0, wheel_version.1
);
@ -553,18 +552,15 @@ fn bytecode_compile(
retries -= 1;
if status.success() || retries == 0 {
break (status, lines);
} else {
warn!(
"Failed to compile {} with python compileall, retrying",
name,
);
}
warn!("Failed to compile {name} with python compileall, retrying",);
};
if !status.success() {
// lossy because we want the error reporting to survive c̴̞̏ü̸̜̹̈́ŕ̴͉̈ś̷̤ė̵̤͋d̷͙̄ filenames in the zip
return Err(Error::PythonSubcommand(io::Error::new(
io::ErrorKind::Other,
format!("Failed to run python compileall, log above: {}", status),
format!("Failed to run python compileall, log above: {status}"),
)));
}
@ -606,7 +602,7 @@ fn bytecode_compile(
path: pyc_path.display().to_string(),
hash: None,
size: None,
})
});
}
Ok(())
@ -655,7 +651,7 @@ fn bytecode_compile_inner(
let line = line.map_err(|err| {
Error::PythonSubcommand(io::Error::new(
io::ErrorKind::Other,
format!("Invalid utf-8 returned by python compileall: {}", err),
format!("Invalid utf-8 returned by python compileall: {err}"),
))
})?;
lines.push(line);
@ -671,17 +667,16 @@ fn bytecode_compile_inner(
///
/// lib/python/site-packages/foo/__init__.py and lib/python/site-packages -> foo/__init__.py
/// lib/marker.txt and lib/python/site-packages -> ../../marker.txt
/// bin/foo_launcher and lib/python/site-packages -> ../../../bin/foo_launcher
/// `bin/foo_launcher` and lib/python/site-packages -> ../../../`bin/foo_launcher`
pub fn relative_to(path: &Path, base: &Path) -> Result<PathBuf, Error> {
// Find the longest common prefix, and also return the path stripped from that prefix
let (stripped, common_prefix) = base
.ancestors()
.filter_map(|ancestor| {
.find_map(|ancestor| {
path.strip_prefix(ancestor)
.ok()
.map(|stripped| (stripped, ancestor))
})
.next()
.ok_or_else(|| {
Error::IO(io::Error::new(
io::ErrorKind::Other,
@ -750,7 +745,7 @@ fn move_folder_recorded(
fn install_script(
site_packages: &Path,
record: &mut [RecordEntry],
file: DirEntry,
file: &DirEntry,
location: &InstallLocation<LockedDir>,
) -> Result<(), Error> {
let path = file.path();
@ -818,7 +813,7 @@ fn install_script(
})?;
entry.path = target_path.display().to_string();
if let Some((size, encoded_hash)) = size_and_encoded_hash {
entry.size = Some(size as usize);
entry.size = Some(size);
entry.hash = Some(encoded_hash);
}
Ok(())
@ -863,7 +858,7 @@ fn install_data(
continue;
}
install_script(site_packages, record, file, &location)?;
install_script(site_packages, record, &file, &location)?;
}
}
Some("headers") => {
@ -911,12 +906,12 @@ fn write_file_recorded(
record.push(RecordEntry {
path: relative_path.display().to_string(),
hash: Some(encoded_hash),
size: Some(content.as_ref().len()),
size: Some(content.as_ref().len() as u64),
});
Ok(())
}
/// Adds INSTALLER, REQUESTED and direct_url.json to the .dist-info dir
/// Adds INSTALLER, REQUESTED and `direct_url.json` to the .dist-info dir
fn extra_dist_info(
site_packages: &Path,
dist_info_prefix: &str,
@ -983,7 +978,7 @@ pub fn parse_key_value_file(
file: &mut impl Read,
debug_filename: &str,
) -> Result<HashMap<String, Vec<String>>, Error> {
let mut data = HashMap::new();
let mut data: HashMap<String, Vec<String>> = HashMap::new();
let file = BufReader::new(file);
for (line_no, line) in file.lines().enumerate() {
@ -993,13 +988,12 @@ pub fn parse_key_value_file(
}
let (key, value) = line.split_once(": ").ok_or_else(|| {
Error::InvalidWheel(format!(
"Line {} of the {} file is invalid",
line_no, debug_filename
"Line {line_no} of the {debug_filename} file is invalid"
))
})?;
data.entry(key.to_string())
.or_insert_with(Vec::new)
.push(value.to_string())
.or_default()
.push(value.to_string());
}
Ok(data)
}
@ -1011,10 +1005,11 @@ pub fn parse_key_value_file(
/// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl>
///
/// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/>
#[allow(clippy::too_many_arguments)]
pub fn install_wheel(
location: &InstallLocation<LockedDir>,
reader: impl Read + Seek,
filename: WheelFilename,
filename: &WheelFilename,
compile: bool,
check_hashes: bool,
// initially used to the console scripts, currently unused. Keeping it because we likely need
@ -1181,11 +1176,11 @@ pub fn install_wheel(
Ok(filename.get_tag())
}
/// From https://github.com/PyO3/python-pkginfo-rs
///
/// The metadata name may be uppercase, while the wheel and dist info names are lowercase, or
/// the metadata name and the dist info name are lowercase, while the wheel name is uppercase.
/// Either way, we just search the wheel for the name
///
/// <https://github.com/PyO3/python-pkginfo-rs>
fn find_dist_info(
filename: &WheelFilename,
archive: &mut ZipArchive<impl Read + Seek + Sized>,
@ -1205,7 +1200,7 @@ fn find_dist_info(
"Missing .dist-info directory".to_string(),
))
}
[dist_info] => dist_info.to_string(),
[dist_info] => (*dist_info).to_string(),
_ => {
return Err(Error::InvalidWheel(format!(
"Multiple .dist-info directories: {}",
@ -1216,7 +1211,7 @@ fn find_dist_info(
Ok(dist_info)
}
/// Adapted from https://github.com/PyO3/python-pkginfo-rs
/// <https://github.com/PyO3/python-pkginfo-rs>
fn read_metadata(
dist_info_prefix: &str,
archive: &mut ZipArchive<impl Read + Seek + Sized>,
@ -1231,50 +1226,45 @@ fn read_metadata(
let mut mail = b"Content-Type: text/plain; charset=utf-8\n".to_vec();
mail.extend_from_slice(&content);
let msg = mailparse::parse_mail(&mail)
.map_err(|err| Error::InvalidWheel(format!("Invalid {}: {}", metadata_file, err)))?;
.map_err(|err| Error::InvalidWheel(format!("Invalid {metadata_file}: {err}")))?;
let headers = msg.get_headers();
let metadata_version =
headers
.get_first_value("Metadata-Version")
.ok_or(Error::InvalidWheel(format!(
"No Metadata-Version field in {}",
metadata_file
"No Metadata-Version field in {metadata_file}"
)))?;
// Crude but it should do https://packaging.python.org/en/latest/specifications/core-metadata/#metadata-version
// At time of writing:
// > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”.
if !(metadata_version.starts_with("1.") || metadata_version.starts_with("2.")) {
return Err(Error::InvalidWheel(format!(
"Metadata-Version field has unsupported value {}",
metadata_version
"Metadata-Version field has unsupported value {metadata_version}"
)));
}
let name = headers
.get_first_value("Name")
.ok_or(Error::InvalidWheel(format!(
"No Name field in {}",
metadata_file
"No Name field in {metadata_file}"
)))?;
let version = headers
.get_first_value("Version")
.ok_or(Error::InvalidWheel(format!(
"No Version field in {}",
metadata_file
"No Version field in {metadata_file}"
)))?;
Ok((name, version))
}
#[cfg(test)]
mod test {
use super::parse_wheel_version;
use crate::wheel::{read_record_file, relative_to};
use crate::{install_wheel, parse_key_value_file, InstallLocation, Script, WheelFilename};
use fs_err as fs;
use std::path::Path;
use indoc::{formatdoc, indoc};
use std::fs::File;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use tempfile::TempDir;
use crate::wheel::{read_record_file, relative_to};
use crate::{parse_key_value_file, Script};
use super::parse_wheel_version;
#[test]
fn test_parse_key_value_file() {
@ -1331,58 +1321,6 @@ mod test {
assert_eq!(expected, actual);
}
/// Previously `__pycache__` paths were erroneously absolute
#[test]
fn installed_paths_relative() {
let filename = "colander-0.9.9-py2.py3-none-any.whl";
let wheel = Path::new("../../test-data/wheels").join(filename);
let temp_dir = TempDir::new().unwrap();
// TODO: Would be nicer to pick the default python here, but i don't want to launch a
// subprocess
let python = if cfg!(target_os = "windows") {
PathBuf::from("python.exe")
} else {
PathBuf::from("python3.8")
};
let install_location = InstallLocation::<PathBuf>::Monotrail {
monotrail_root: temp_dir.path().to_path_buf(),
python: python.clone(),
python_version: (3, 8),
}
.acquire_lock()
.unwrap();
install_wheel(
&install_location,
File::open(wheel).unwrap(),
WheelFilename::from_str(&filename).unwrap(),
true,
true,
&[],
"0.9.9",
&python,
)
.unwrap();
let base = temp_dir
.path()
.join("colander")
.join("0.9.9")
.join("py2.py3-none-any");
let mid = if cfg!(windows) {
base.join("Lib")
} else {
base.join("lib").join("python")
};
let record = mid
.join("site-packages")
.join("colander-0.9.9.dist-info")
.join("RECORD");
let record = fs::read_to_string(&record).unwrap();
for line in record.lines() {
assert!(!line.starts_with('/'), "{}", line);
}
}
#[test]
fn test_relative_to() {
assert_eq!(

937
crates/install-wheel-rs/src/wheel_tags.rs
View file

@ -1,937 +0,0 @@
//! Parses the wheel filename, the current host os/arch and checks wheels for compatibility
use crate::Error;
use fs_err as fs;
use goblin::elf::Elf;
use once_cell::sync::Lazy;
use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI};
use regex::Regex;
use serde::Deserialize;
use std::fmt;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::str::FromStr;
use tracing::trace;
/// The name of a wheel split into its parts ([PEP 491](https://peps.python.org/pep-0491/))
///
/// Ignores the build tag atm.
///
/// ```
/// use std::str::FromStr;
/// use install_wheel_rs::WheelFilename;
///
/// let filename = WheelFilename::from_str("foo-1.0-py32-none-any.whl").unwrap();
/// assert_eq!(filename, WheelFilename {
/// distribution: "foo".to_string(),
/// version: "1.0".to_string(),
/// python_tag: vec!["py32".to_string()],
/// abi_tag: vec!["none".to_string()],
/// platform_tag: vec!["any".to_string()]
/// });
/// let filename = WheelFilename::from_str(
/// "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
/// ).unwrap();
/// assert_eq!(filename, WheelFilename {
/// distribution: "numpy".to_string(),
/// version: "1.26.0".to_string(),
/// python_tag: vec!["cp312".to_string()],
/// abi_tag: vec!["cp312".to_string()],
/// platform_tag: vec![
/// "manylinux_2_17_aarch64".to_string(),
/// "manylinux2014_aarch64".to_string()
/// ]
/// });
/// ```
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WheelFilename {
pub distribution: String,
pub version: String,
pub python_tag: Vec<String>,
pub abi_tag: Vec<String>,
pub platform_tag: Vec<String>,
}
impl FromStr for WheelFilename {
type Err = Error;
fn from_str(filename: &str) -> Result<Self, Self::Err> {
let basename = filename.strip_suffix(".whl").ok_or_else(|| {
Error::InvalidWheelFileName(filename.to_string(), "Must end with .whl".to_string())
})?;
// https://www.python.org/dev/peps/pep-0427/#file-name-convention
match basename.split('-').collect::<Vec<_>>().as_slice() {
// TODO: Build tag precedence
&[distribution, version, _, python_tag, abi_tag, platform_tag]
| &[distribution, version, python_tag, abi_tag, platform_tag] => Ok(WheelFilename {
distribution: distribution.to_string(),
version: version.to_string(),
python_tag: python_tag.split('.').map(String::from).collect(),
abi_tag: abi_tag.split('.').map(String::from).collect(),
platform_tag: platform_tag.split('.').map(String::from).collect(),
}),
_ => Err(Error::InvalidWheelFileName(
filename.to_string(),
"Expected four or five dashes (\"-\") in the filename".to_string(),
)),
}
}
}
impl WheelFilename {
/// Returns Some(precedence) is the wheels are compatible, otherwise none
///
/// Precedence is e.g. used to install newer manylinux wheels over older manylinux wheels
pub fn compatibility(&self, compatible_tags: &CompatibleTags) -> Result<usize, Error> {
compatible_tags
.iter()
.enumerate()
.filter_map(|(precedence, tag)| {
if self.python_tag.contains(&tag.0)
&& self.abi_tag.contains(&tag.1)
&& self.platform_tag.contains(&tag.2)
{
Some(precedence)
} else {
None
}
})
.next()
.ok_or_else(|| Error::IncompatibleWheel {
os: compatible_tags.os.clone(),
arch: compatible_tags.arch,
})
}
/// Effectively undoes the wheel filename parsing step
pub fn get_tag(&self) -> String {
format!(
"{}-{}-{}",
self.python_tag.join("."),
self.abi_tag.join("."),
self.platform_tag.join(".")
)
}
}
/// A platform, defined by the list of compatible wheel tags in order
pub struct CompatibleTags {
pub os: Os,
pub arch: Arch,
pub tags: Vec<(String, String, String)>,
}
impl Deref for CompatibleTags {
type Target = [(String, String, String)];
fn deref(&self) -> &Self::Target {
&self.tags
}
}
/// Returns the compatible tags in a (python_tag, abi_tag, platform_tag) format, ordered from
/// highest precedence to lowest precedence
impl CompatibleTags {
/// Compatible tags for the current operating system and architecture
pub fn current(python_version: (u8, u8)) -> Result<CompatibleTags, Error> {
Self::new(python_version, Os::current()?, Arch::current()?)
}
pub fn new(python_version: (u8, u8), os: Os, arch: Arch) -> Result<CompatibleTags, Error> {
assert_eq!(python_version.0, 3);
let mut tags = Vec::new();
let platform_tags = compatible_platform_tags(&os, &arch)?;
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
Ok(CompatibleTags { os, arch, tags })
}
}
/// All supported operating system
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum Os {
Manylinux { major: u16, minor: u16 },
Musllinux { major: u16, minor: u16 },
Windows,
Macos { major: u16, minor: u16 },
FreeBsd { release: String },
NetBsd { release: String },
OpenBsd { release: String },
Dragonfly { release: String },
Illumos { release: String, arch: String },
Haiku { release: String },
}
impl Os {
fn detect_linux_libc() -> Result<Self, Error> {
let libc = find_libc()?;
let linux = if let Ok(Some((major, minor))) = get_musl_version(&libc) {
Os::Musllinux { major, minor }
} else if let Ok(glibc_ld) = fs::read_link(&libc) {
// Try reading the link first as it's faster
let filename = glibc_ld
.file_name()
.ok_or_else(|| {
Error::OsVersionDetection("Expected the glibc ld to be a file".to_string())
})?
.to_string_lossy();
#[allow(non_upper_case_globals)]
static expr: Lazy<Regex> =
Lazy::new(|| Regex::new(r"ld-(\d{1,3})\.(\d{1,3})\.so").unwrap());
if let Some(capture) = expr.captures(&filename) {
let major = capture.get(1).unwrap().as_str().parse::<u16>().unwrap();
let minor = capture.get(2).unwrap().as_str().parse::<u16>().unwrap();
Os::Manylinux { major, minor }
} else {
trace!("Couldn't use ld filename, using `ldd --version`");
// runs `ldd --version`
let version = glibc_version::get_version().map_err(|err| {
Error::OsVersionDetection(format!(
"Failed to determine glibc version with `ldd --version`: {}",
err
))
})?;
Os::Manylinux {
major: version.major as u16,
minor: version.minor as u16,
}
}
} else {
return Err(Error::OsVersionDetection("Couldn't detect neither glibc version nor musl libc version, at least one of which is required".to_string()));
};
trace!("libc: {}", linux);
Ok(linux)
}
pub fn current() -> Result<Self, Error> {
let target_triple = target_lexicon::HOST;
let os = match target_triple.operating_system {
target_lexicon::OperatingSystem::Linux => Self::detect_linux_libc()?,
target_lexicon::OperatingSystem::Windows => Os::Windows,
target_lexicon::OperatingSystem::MacOSX { major, minor, .. } => {
Os::Macos { major, minor }
}
target_lexicon::OperatingSystem::Darwin => {
let (major, minor) = get_mac_os_version()?;
Os::Macos { major, minor }
}
target_lexicon::OperatingSystem::Netbsd => Os::NetBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Freebsd => Os::FreeBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Openbsd => Os::OpenBsd {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Dragonfly => Os::Dragonfly {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
target_lexicon::OperatingSystem::Illumos => {
let platform_info = PlatformInfo::new().map_err(Error::PlatformInfo)?;
Os::Illumos {
release: platform_info.release().to_string_lossy().to_string(),
arch: platform_info.machine().to_string_lossy().to_string(),
}
}
target_lexicon::OperatingSystem::Haiku => Os::Haiku {
release: PlatformInfo::new()
.map_err(Error::PlatformInfo)?
.release()
.to_string_lossy()
.to_string(),
},
unsupported => {
return Err(Error::OsVersionDetection(format!(
"The operating system {:?} is not supported",
unsupported
)))
}
};
Ok(os)
}
}
impl fmt::Display for Os {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Os::Manylinux { .. } => write!(f, "Manylinux"),
Os::Musllinux { .. } => write!(f, "Musllinux"),
Os::Windows => write!(f, "Windows"),
Os::Macos { .. } => write!(f, "MacOS"),
Os::FreeBsd { .. } => write!(f, "FreeBSD"),
Os::NetBsd { .. } => write!(f, "NetBSD"),
Os::OpenBsd { .. } => write!(f, "OpenBSD"),
Os::Dragonfly { .. } => write!(f, "DragonFly"),
Os::Illumos { .. } => write!(f, "Illumos"),
Os::Haiku { .. } => write!(f, "Haiku"),
}
}
}
/// All supported CPU architectures
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum Arch {
Aarch64,
Armv7L,
Powerpc64Le,
Powerpc64,
X86,
X86_64,
S390X,
}
impl fmt::Display for Arch {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Arch::Aarch64 => write!(f, "aarch64"),
Arch::Armv7L => write!(f, "armv7l"),
Arch::Powerpc64Le => write!(f, "ppc64le"),
Arch::Powerpc64 => write!(f, "ppc64"),
Arch::X86 => write!(f, "i686"),
Arch::X86_64 => write!(f, "x86_64"),
Arch::S390X => write!(f, "s390x"),
}
}
}
impl Arch {
pub fn current() -> Result<Arch, Error> {
let target_triple = target_lexicon::HOST;
let arch = match target_triple.architecture {
target_lexicon::Architecture::X86_64 => Arch::X86_64,
target_lexicon::Architecture::X86_32(_) => Arch::X86,
target_lexicon::Architecture::Arm(_) => Arch::Armv7L,
target_lexicon::Architecture::Aarch64(_) => Arch::Aarch64,
target_lexicon::Architecture::Powerpc64 => Arch::Powerpc64,
target_lexicon::Architecture::Powerpc64le => Arch::Powerpc64Le,
target_lexicon::Architecture::S390x => Arch::S390X,
unsupported => {
return Err(Error::OsVersionDetection(format!(
"The architecture {} is not supported",
unsupported
)));
}
};
Ok(arch)
}
/// Returns the oldest possible Manylinux tag for this architecture
pub fn get_minimum_manylinux_minor(&self) -> u16 {
match self {
// manylinux 2014
Arch::Aarch64 | Arch::Armv7L | Arch::Powerpc64 | Arch::Powerpc64Le | Arch::S390X => 17,
// manylinux 1
Arch::X86 | Arch::X86_64 => 5,
}
}
}
fn get_mac_os_version() -> Result<(u16, u16), Error> {
// This is actually what python does
// https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct SystemVersion {
product_version: String,
}
let system_version: SystemVersion =
plist::from_file("/System/Library/CoreServices/SystemVersion.plist")
.map_err(|err| Error::OsVersionDetection(err.to_string()))?;
let invalid_mac_os_version = || {
Error::OsVersionDetection(format!(
"Invalid mac os version {}",
system_version.product_version
))
};
match system_version
.product_version
.split('.')
.collect::<Vec<&str>>()
.as_slice()
{
[major, minor] | [major, minor, _] => {
let major = major.parse::<u16>().map_err(|_| invalid_mac_os_version())?;
let minor = minor.parse::<u16>().map_err(|_| invalid_mac_os_version())?;
Ok((major, minor))
}
_ => Err(invalid_mac_os_version()),
}
}
/// Determine the appropriate binary formats for a mac os version.
/// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314
fn get_mac_binary_formats(major: u16, minor: u16, arch: &Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}
/// Find musl libc path from executable's ELF header
pub fn find_libc() -> Result<PathBuf, Error> {
let buffer = fs::read("/bin/ls")?;
let error_str = "Couldn't parse /bin/ls for detecting the ld version";
let elf = Elf::parse(&buffer)
.map_err(|err| Error::OsVersionDetection(format!("{}: {}", error_str, err)))?;
if let Some(elf_interpreter) = elf.interpreter {
Ok(PathBuf::from(elf_interpreter))
} else {
Err(Error::OsVersionDetection(error_str.to_string()))
}
}
/// Read the musl version from libc library's output. Taken from maturin
///
/// The libc library should output something like this to stderr::
///
/// musl libc (x86_64)
/// Version 1.2.2
/// Dynamic Program Loader
pub fn get_musl_version(ld_path: impl AsRef<Path>) -> std::io::Result<Option<(u16, u16)>> {
let output = Command::new(ld_path.as_ref())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()?;
let stderr = String::from_utf8_lossy(&output.stderr);
#[allow(non_upper_case_globals)]
static expr: Lazy<Regex> = Lazy::new(|| Regex::new(r"Version (\d{2,4})\.(\d{2,4})").unwrap());
if let Some(capture) = expr.captures(&stderr) {
let major = capture.get(1).unwrap().as_str().parse::<u16>().unwrap();
let minor = capture.get(2).unwrap().as_str().parse::<u16>().unwrap();
return Ok(Some((major, minor)));
}
Ok(None)
}
/// Returns the compatible platform tags from highest precedence to lowest precedence
///
/// Examples: manylinux_2_17, macosx_11_0_arm64, win_amd64
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up. The order also isn't exactly matching that of pip yet,
/// but works good enough in practice
pub fn compatible_platform_tags(os: &Os, arch: &Arch) -> Result<Vec<String>, Error> {
let platform_tags = match (os.clone(), *arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// Use newer manylinux first like pip does
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=minor)
.rev()
.map(|minor| format!("manylinux_{}_{}_{}", major, minor, arch)),
);
if (arch.get_minimum_manylinux_minor()..=minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{}", arch))
}
if (arch.get_minimum_manylinux_minor()..=minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{}", arch))
}
if (arch.get_minimum_manylinux_minor()..=minor).contains(&5) {
platform_tags.push(format!("manylinux1_{}", arch))
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags.extend(
(1..=minor)
.rev()
.map(|minor| format!("musllinux_{}_{}_{}", major, minor, arch)),
);
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=minor).rev() {
for binary_format in get_mac_binary_formats(major, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", major, minor, binary_format));
}
}
}
x if x >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(Error::OsVersionDetection(format!(
"Unsupported mac os version: {}",
major,
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release: _ }
| Os::NetBsd { release: _ }
| Os::OpenBsd { release: _ }
| Os::Dragonfly { release: _ }
| Os::Haiku { release: _ },
_,
) => {
let info = PlatformInfo::new().map_err(Error::PlatformInfo)?;
let release = info.release().to_string_lossy().replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(
Os::Illumos {
mut release,
mut arch,
},
_,
) => {
let mut os = os.to_string().to_lowercase();
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
Error::OsVersionDetection(format!(
"illumos major version is not a number: {}",
err
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
os = "solaris".to_string();
release = format!("{}_{}", major_ver - 3, other);
arch = format!("{}_64bit", arch);
}
}
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(Error::OsVersionDetection(format!(
"Unsupported operating system and architecture combination: {} {}",
os, arch
)));
}
};
Ok(platform_tags)
}
#[cfg(test)]
mod test {
use super::{compatible_platform_tags, WheelFilename};
use crate::{Arch, CompatibleTags, Error, Os};
use fs_err::File;
use std::str::FromStr;
const FILENAMES: &[&str] = &[
"numpy-1.22.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp310-cp310-win_amd64.whl",
"numpy-1.22.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp310-cp310-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp310-cp310-macosx_10_14_x86_64.whl",
"numpy-1.22.2-cp39-cp39-win_amd64.whl",
"numpy-1.22.2-cp39-cp39-win32.whl",
"numpy-1.22.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp39-cp39-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp39-cp39-macosx_10_14_x86_64.whl",
"numpy-1.22.2-cp38-cp38-win_amd64.whl",
"numpy-1.22.2-cp38-cp38-win32.whl",
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl",
"numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl",
"tqdm-4.62.3-py2.py3-none-any.whl",
];
/// Test that we can parse the filenames
#[test]
fn test_wheel_filename_parsing() -> Result<(), Error> {
for filename in FILENAMES {
WheelFilename::from_str(filename)?;
}
Ok(())
}
/// Test that we correctly identify compatible pairs
#[test]
fn test_compatibility() -> Result<(), Error> {
let filenames = [
(
"numpy-1.22.2-cp38-cp38-win_amd64.whl",
((3, 8), Os::Windows, Arch::X86_64),
),
(
"numpy-1.22.2-cp38-cp38-win32.whl",
((3, 8), Os::Windows, Arch::X86),
),
(
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
),
),
(
"numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::Aarch64,
),
),
(
"numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl",
(
(3, 8),
Os::Macos {
major: 11,
minor: 0,
},
Arch::Aarch64,
),
),
(
"numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl",
(
(3, 8),
// Test backwards compatibility here
Os::Macos {
major: 11,
minor: 0,
},
Arch::X86_64,
),
),
(
"ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl",
(
(3, 8),
Os::Macos {
major: 12,
minor: 0,
},
Arch::X86_64,
),
),
(
"ruff-0.0.63-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl",
(
(3, 8),
Os::Macos {
major: 12,
minor: 0,
},
Arch::Aarch64,
),
),
(
"tqdm-4.62.3-py2.py3-none-any.whl",
(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
),
),
];
for (filename, (python_version, os, arch)) in filenames {
let compatible_tags = CompatibleTags::new(python_version, os, arch)?;
assert!(
WheelFilename::from_str(filename)?
.compatibility(&compatible_tags)
.is_ok(),
"{}",
filename
);
}
Ok(())
}
/// Test that incompatible pairs don't pass is_compatible
#[test]
fn test_compatibility_filter() -> Result<(), Error> {
let compatible_tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?;
let compatible: Vec<&str> = FILENAMES
.iter()
.filter(|filename| {
WheelFilename::from_str(filename)
.unwrap()
.compatibility(&compatible_tags)
.is_ok()
})
.cloned()
.collect();
assert_eq!(
vec![
"numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"tqdm-4.62.3-py2.py3-none-any.whl"
],
compatible
);
Ok(())
}
fn get_ubuntu_20_04_tags() -> Vec<String> {
serde_json::from_reader(File::open("../../test-data/tags/cp38-ubuntu-20-04.json").unwrap())
.unwrap()
}
/// Check against the tags that packaging.tags reports as compatible
#[test]
fn ubuntu_20_04_compatible() -> Result<(), Error> {
let tags = get_ubuntu_20_04_tags();
for tag in tags {
let compatible_tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?;
assert!(
WheelFilename::from_str(&format!("foo-1.0-{}.whl", tag))?
.compatibility(&compatible_tags)
.is_ok(),
"{}",
tag
)
}
Ok(())
}
/// Check against the tags that packaging.tags reports as compatible
#[test]
fn ubuntu_20_04_list() -> Result<(), Error> {
let expected_tags = get_ubuntu_20_04_tags();
let actual_tags: Vec<String> = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)?
.iter()
.map(|(python_tag, abi_tag, platform_tag)| {
format!("{}-{}-{}", python_tag, abi_tag, platform_tag)
})
.collect();
assert_eq!(expected_tags, actual_tags);
Ok(())
}
#[test]
fn test_precedence() {
let tags = CompatibleTags::new(
(3, 8),
Os::Manylinux {
major: 2,
minor: 31,
},
Arch::X86_64,
)
.unwrap();
let pairs = [
(
"greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl"
),
(
"regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"regex-2022.10.31-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl"
),
];
for (higher_str, lower_str) in pairs {
let higher = WheelFilename::from_str(higher_str).unwrap();
let lower = WheelFilename::from_str(lower_str).unwrap();
let higher_precedence = higher.compatibility(&tags).unwrap();
let lower_precedence = lower.compatibility(&tags).unwrap();
assert!(
higher_precedence < lower_precedence,
"{} {} {} {}",
higher_str,
higher_precedence,
lower_str,
lower_precedence
);
}
}
/// Basic does-it-work test
#[test]
fn host_arch() -> Result<(), Error> {
let os = Os::current()?;
let arch = Arch::current()?;
compatible_platform_tags(&os, &arch)?;
Ok(())
}
}

BIN
crates/install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl
View file

Binary file not shown.

35
crates/install-wheel-rs/test/test_install_wheel_rs.py
View file

@ -1,35 +0,0 @@
import platform
from pathlib import Path
from subprocess import check_call, check_output
def check_installed(venv: Path) -> bool:
"""true: installed and working, false: not installed, borked: exception"""
try:
output = check_output(
[
venv.joinpath(
"Scripts" if platform.system() == "Windows" else "bin"
).joinpath("upsidedown")
],
input="hello world!",
text=True,
).strip()
except FileNotFoundError:
return False
assert output == "¡pꞁɹoʍ oꞁꞁǝɥ"
return True
def test_install_wheel_rs(pytestconfig, tmp_path):
from install_wheel_rs import LockedVenv
venv = tmp_path.joinpath("venv_test_install_wheel_rs")
check_call(["virtualenv", venv])
assert not check_installed(venv)
locked_venv = LockedVenv(venv)
wheel = pytestconfig.rootpath.joinpath(
"install-wheel-rs/test-data/upsidedown-0.4-py2.py3-none-any.whl"
)
locked_venv.install_wheel(wheel)
assert check_installed(venv)

5
crates/puffin-platform/Cargo.toml → crates/platform-host/Cargo.toml
View file

@ -1,5 +1,5 @@
[package]
name = "puffin-platform"
name = "platform-host"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
@ -10,9 +10,10 @@ authors.workspace = true
license.workspace = true
[dependencies]
pep440_rs = { path = "../pep440-rs" }
glibc_version = { workspace = true }
goblin = { workspace = true }
pep440_rs = { path = "../pep440-rs" }
platform-info = { workspace = true }
plist = { workspace = true }
regex = { workspace = true }

10
crates/puffin-platform/src/lib.rs → crates/platform-host/src/lib.rs
View file

@ -11,8 +11,6 @@ use serde::Deserialize;
use thiserror::Error;
use tracing::trace;
pub mod tags;
#[derive(Error, Debug)]
pub enum PlatformError {
#[error(transparent)]
@ -34,8 +32,12 @@ impl Platform {
Ok(Self { os, arch })
}
pub fn is_windows(&self) -> bool {
matches!(self.os, Os::Windows)
pub fn os(&self) -> &Os {
&self.os
}
pub fn arch(&self) -> Arch {
self.arch
}
}

13
crates/platform-tags/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "platform-tags"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
platform-host = { path = "../platform-host" }

258
crates/platform-tags/src/lib.rs Normal file
View file

@ -0,0 +1,258 @@
use platform_host::{Arch, Os, Platform, PlatformError};
/// A set of compatible tags for a given Python version and platform, in
/// (`python_tag`, `abi_tag`, `platform_tag`) format.
#[derive(Debug)]
pub struct Tags(Vec<(String, String, String)>);
impl Tags {
/// Returns the compatible tags for the given Python version and platform.
pub fn from_env(platform: &Platform, python_version: (u8, u8)) -> Result<Self, PlatformError> {
let platform_tags = compatible_tags(platform)?;
let mut tags = Vec::with_capacity(5 * platform_tags.len());
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
tags.sort();
Ok(Self(tags))
}
pub fn iter(&self) -> impl Iterator<Item = &(String, String, String)> {
self.0.iter()
}
}
/// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`,
/// `macosx_11_0_arm64`, or `win_amd64`).
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up.
fn compatible_tags(platform: &Platform) -> Result<Vec<String>, PlatformError> {
let os = platform.os();
let arch = platform.arch();
let platform_tags = match (&os, arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=*minor)
.map(|minor| format!("manylinux_{major}_{minor}_{arch}")),
);
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) {
platform_tags.push(format!("manylinux1_{arch}"));
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags
.extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}")));
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=*minor).rev() {
for binary_format in get_mac_binary_formats(*major, minor, arch) {
platform_tags.push(format!("macosx_{major}_{minor}_{binary_format}"));
}
}
}
value if *value >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported macOS version: {major}",
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release }
| Os::NetBsd { release }
| Os::OpenBsd { release }
| Os::Dragonfly { release }
| Os::Haiku { release },
_,
) => {
let release = release.replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(Os::Illumos { release, arch }, _) => {
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
PlatformError::OsVersionDetectionError(format!(
"illumos major version is not a number: {err}"
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
let os = "solaris".to_string();
let release = format!("{}_{}", major_ver - 3, other);
let arch = format!("{arch}_64bit");
return Ok(vec![format!("{}_{}_{}", os, release, arch)]);
}
}
let os = os.to_string().to_lowercase();
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported operating system and architecture combination: {os} {arch}"
)));
}
};
Ok(platform_tags)
}
/// Determine the appropriate binary formats for a macOS version.
/// Source: <https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314>
fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}

8
crates/puffin-cli/Cargo.toml
View file

@ -4,11 +4,14 @@ version = "0.1.0"
edition = "2021"
[dependencies]
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
platform-tags = { path = "../platform-tags" }
puffin-client = { path = "../puffin-client" }
puffin-installer = { path = "../puffin-installer" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" }
platform-host = { path = "../platform-host" }
puffin-resolver = { path = "../puffin-resolver" }
anyhow = { workspace = true }
@ -16,9 +19,6 @@ clap = { workspace = true, features = ["derive"] }
colored = { workspace = true }
directories = { workspace = true }
futures = { workspace = true }
install-wheel-rs = { workspace = true }
pep508_rs = { path = "../pep508-rs" }
pep440_rs = { path = "../pep440-rs" }
tracing = { workspace = true }
tracing-tree = { workspace = true }
tracing-subscriber = { workspace = true }

6
crates/puffin-cli/src/commands/compile.rs
View file

@ -4,10 +4,10 @@ use std::str::FromStr;
use anyhow::Result;
use tracing::debug;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags;
use puffin_platform::Platform;
use crate::commands::ExitStatus;
@ -31,7 +31,7 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result<ExitStat
let markers = python.markers();
// Determine the compatible platform tags.
let tags = Tags::from_env(&platform, python.version())?;
let tags = Tags::from_env(&platform, python.simple_version())?;
// Instantiate a client.
let client = {

6
crates/puffin-cli/src/commands/sync.rs
View file

@ -4,10 +4,10 @@ use std::str::FromStr;
use anyhow::Result;
use tracing::debug;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags;
use puffin_platform::Platform;
use crate::commands::ExitStatus;
@ -31,7 +31,7 @@ pub(crate) async fn sync(src: &Path, cache: Option<&Path>) -> Result<ExitStatus>
let markers = python.markers();
// Determine the compatible platform tags.
let tags = Tags::from_env(&platform, python.version())?;
let tags = Tags::from_env(&platform, python.simple_version())?;
// Instantiate a client.
let client = {

3
crates/puffin-installer/Cargo.toml
View file

@ -10,11 +10,12 @@ authors.workspace = true
license.workspace = true
[dependencies]
install-wheel-rs = { path = "../install-wheel-rs", default-features = false }
puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" }
wheel-filename = { path = "../wheel-filename" }
anyhow = { workspace = true }
install-wheel-rs = { workspace = true }
tempfile = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }

8
crates/puffin-installer/src/lib.rs
View file

@ -2,13 +2,14 @@ use std::path::Path;
use std::str::FromStr;
use anyhow::Result;
use install_wheel_rs::{install_wheel, InstallLocation};
use tokio::task::JoinSet;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use url::Url;
use install_wheel_rs::{install_wheel, InstallLocation};
use puffin_client::{File, PypiClient};
use puffin_interpreter::PythonExecutable;
use wheel_filename::WheelFilename;
/// Install a set of wheels into a Python virtual environment.
pub async fn install(
@ -40,13 +41,14 @@ pub async fn install(
let locked_dir = location.acquire_lock()?;
for wheel in wheels {
let path = tmp_dir.path().join(&wheel.hashes.sha256);
let filename = install_wheel_rs::WheelFilename::from_str(&wheel.filename)?;
let filename = WheelFilename::from_str(&wheel.filename)?;
// TODO(charlie): Should this be async?
install_wheel(
&locked_dir,
std::fs::File::open(path)?,
filename,
&filename,
false,
false,
&[],
"",

2
crates/puffin-interpreter/Cargo.toml
View file

@ -12,7 +12,7 @@ license.workspace = true
[dependencies]
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
puffin-platform = { path = "../puffin-platform" }
platform-host = { path = "../platform-host" }
anyhow = { workspace = true }
serde_json = { workspace = true }

4
crates/puffin-interpreter/src/lib.rs
View file

@ -1,10 +1,10 @@
use std::path::{Path, PathBuf};
use anyhow::Result;
use pep440_rs::Version;
use pep508_rs::MarkerEnvironment;
use puffin_platform::Platform;
use platform_host::Platform;
use crate::python_platform::PythonPlatform;

1
crates/puffin-interpreter/src/markers.rs
View file

@ -3,6 +3,7 @@ use std::path::Path;
use std::process::{Command, Output};
use anyhow::{Context, Result};
use pep508_rs::MarkerEnvironment;
/// Return the resolved [`MarkerEnvironment`] for the given Python executable.

6
crates/puffin-interpreter/src/python_platform.rs
View file

@ -1,7 +1,7 @@
use std::path::Path;
use std::path::PathBuf;
use puffin_platform::Platform;
use platform_host::{Os, Platform};
/// A Python-aware wrapper around [`Platform`].
#[derive(Debug, Clone, Eq, PartialEq)]
@ -10,7 +10,7 @@ pub(crate) struct PythonPlatform<'a>(&'a Platform);
impl PythonPlatform<'_> {
/// Returns the path to the `python` executable inside a virtual environment.
pub(crate) fn venv_python(&self, venv_base: impl AsRef<Path>) -> PathBuf {
let python = if self.0.is_windows() {
let python = if matches!(self.0.os(), Os::Windows) {
"python.exe"
} else {
"python"
@ -21,7 +21,7 @@ impl PythonPlatform<'_> {
/// Returns the directory in which the binaries are stored inside a virtual environment.
pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef<Path>) -> PathBuf {
let venv = venv_base.as_ref();
if self.0.is_windows() {
if matches!(self.0.os(), Os::Windows) {
let bin_dir = venv.join("Scripts");
if bin_dir.join("python.exe").exists() {
return bin_dir;

2
crates/puffin-package/Cargo.toml
View file

@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
pep440_rs = { path = "../pep440-rs", features = ["serde"] }
pep508_rs = { path = "../pep508-rs", features = ["serde"] }
puffin-platform = { path = "../puffin-platform" }
platform-host = { path = "../platform-host" }
anyhow = { workspace = true }
mailparse = { workspace = true }

1
crates/puffin-package/src/lib.rs
View file

@ -1,4 +1,3 @@
pub mod metadata;
pub mod package_name;
pub mod requirements;
pub mod wheel;

5
crates/puffin-package/src/metadata.rs
View file

@ -5,11 +5,12 @@ use std::io;
use std::str::FromStr;
use mailparse::{MailHeaderMap, MailParseError};
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
/// Python Package Metadata 2.1 as specified in
/// <https://packaging.python.org/specifications/core-metadata/>
///

1
crates/puffin-package/src/requirements.rs
View file

@ -4,6 +4,7 @@ use std::str::FromStr;
use anyhow::Result;
use memchr::{memchr2, memchr_iter};
use pep508_rs::{Pep508Error, Requirement};
#[derive(Debug)]

267
crates/puffin-platform/src/tags.rs
View file

@ -1,267 +0,0 @@
use crate::{Arch, Os, Platform, PlatformError};
/// A set of compatible tags for a given Python version and platform, in
/// (`python_tag`, `abi_tag`, `platform_tag`) format.
#[derive(Debug)]
pub struct Tags(Vec<(String, String, String)>);
impl Tags {
/// Returns the compatible tags for the given Python version and platform.
pub fn from_env(
platform: &Platform,
python_version: &pep440_rs::Version,
) -> Result<Self, PlatformError> {
let python_version = (python_version.release[0], python_version.release[1]);
let platform_tags = platform.compatible_tags()?;
let mut tags = Vec::with_capacity(5 * platform_tags.len());
// 1. This exact c api version
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
format!(
"cp{}{}{}",
python_version.0,
python_version.1,
// hacky but that's legacy anyways
if python_version.1 <= 7 { "m" } else { "" }
),
platform_tag.clone(),
));
tags.push((
format!("cp{}{}", python_version.0, python_version.1),
"none".to_string(),
platform_tag.clone(),
));
}
// 2. abi3 and no abi (e.g. executable binary)
// For some reason 3.2 is the minimum python for the cp abi
for minor in 2..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("cp{}{}", python_version.0, minor),
"abi3".to_string(),
platform_tag.clone(),
));
}
}
// 3. no abi (e.g. executable binary)
for minor in 0..=python_version.1 {
for platform_tag in &platform_tags {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
platform_tag.clone(),
));
}
}
// 4. major only
for platform_tag in platform_tags {
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
platform_tag,
));
}
// 5. no binary
for minor in 0..=python_version.1 {
tags.push((
format!("py{}{}", python_version.0, minor),
"none".to_string(),
"any".to_string(),
));
}
tags.push((
format!("py{}", python_version.0),
"none".to_string(),
"any".to_string(),
));
tags.sort();
Ok(Self(tags))
}
pub fn iter(&self) -> impl Iterator<Item = &(String, String, String)> {
self.0.iter()
}
}
impl Platform {
/// Returns the compatible tags for the current [`Platform`] (e.g., `manylinux_2_17`,
/// `macosx_11_0_arm64`, or `win_amd64`).
///
/// We have two cases: Actual platform specific tags (including "merged" tags such as universal2)
/// and "any".
///
/// Bit of a mess, needs to be cleaned up.
fn compatible_tags(&self) -> Result<Vec<String>, PlatformError> {
let os = &self.os;
let arch = self.arch;
let platform_tags = match (&os, arch) {
(Os::Manylinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
platform_tags.extend(
(arch.get_minimum_manylinux_minor()..=*minor)
.map(|minor| format!("manylinux_{major}_{minor}_{arch}")),
);
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&12) {
platform_tags.push(format!("manylinux2010_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&17) {
platform_tags.push(format!("manylinux2014_{arch}"));
}
if (arch.get_minimum_manylinux_minor()..=*minor).contains(&5) {
platform_tags.push(format!("manylinux1_{arch}"));
}
platform_tags
}
(Os::Musllinux { major, minor }, _) => {
let mut platform_tags = vec![format!("linux_{}", arch)];
// musl 1.1 is the lowest supported version in musllinux
platform_tags
.extend((1..=*minor).map(|minor| format!("musllinux_{major}_{minor}_{arch}")));
platform_tags
}
(Os::Macos { major, minor }, Arch::X86_64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
match major {
10 => {
// Prior to Mac OS 11, each yearly release of Mac OS bumped the "minor" version
// number. The major version was always 10.
for minor in (0..=*minor).rev() {
for binary_format in get_mac_binary_formats(*major, minor, arch) {
platform_tags
.push(format!("macosx_{major}_{minor}_{binary_format}"));
}
}
}
value if *value >= 11 => {
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
for minor in (4..=16).rev() {
for binary_format in get_mac_binary_formats(10, minor, arch) {
platform_tags
.push(format!("macosx_{}_{}_{}", 10, minor, binary_format));
}
}
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported macOS version: {major}",
)));
}
}
platform_tags
}
(Os::Macos { major, .. }, Arch::Aarch64) => {
// Source: https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L346
let mut platform_tags = vec![];
// Starting with Mac OS 11, each yearly release bumps the major version number.
// The minor versions are now the midyear updates.
for major in (10..=*major).rev() {
for binary_format in get_mac_binary_formats(major, 0, arch) {
platform_tags.push(format!("macosx_{}_{}_{}", major, 0, binary_format));
}
}
// The "universal2" binary format can have a macOS version earlier than 11.0
// when the x86_64 part of the binary supports that version of macOS.
platform_tags.extend(
(4..=16)
.rev()
.map(|minor| format!("macosx_{}_{}_universal2", 10, minor)),
);
platform_tags
}
(Os::Windows, Arch::X86) => {
vec!["win32".to_string()]
}
(Os::Windows, Arch::X86_64) => {
vec!["win_amd64".to_string()]
}
(Os::Windows, Arch::Aarch64) => vec!["win_arm64".to_string()],
(
Os::FreeBsd { release }
| Os::NetBsd { release }
| Os::OpenBsd { release }
| Os::Dragonfly { release }
| Os::Haiku { release },
_,
) => {
let release = release.replace(['.', '-'], "_");
vec![format!(
"{}_{}_{}",
os.to_string().to_lowercase(),
release,
arch
)]
}
(Os::Illumos { release, arch }, _) => {
// See https://github.com/python/cpython/blob/46c8d915715aa2bd4d697482aa051fe974d440e1/Lib/sysconfig.py#L722-L730
if let Some((major, other)) = release.split_once('_') {
let major_ver: u64 = major.parse().map_err(|err| {
PlatformError::OsVersionDetectionError(format!(
"illumos major version is not a number: {err}"
))
})?;
if major_ver >= 5 {
// SunOS 5 == Solaris 2
let os = "solaris".to_string();
let release = format!("{}_{}", major_ver - 3, other);
let arch = format!("{arch}_64bit");
return Ok(vec![format!("{}_{}_{}", os, release, arch)]);
}
}
let os = os.to_string().to_lowercase();
vec![format!("{}_{}_{}", os, release, arch)]
}
_ => {
return Err(PlatformError::OsVersionDetectionError(format!(
"Unsupported operating system and architecture combination: {os} {arch}"
)));
}
};
Ok(platform_tags)
}
}
/// Determine the appropriate binary formats for a macOS version.
/// Source: <https://github.com/pypa/packaging/blob/fd4f11139d1c884a637be8aa26bb60a31fbc9411/packaging/tags.py#L314>
fn get_mac_binary_formats(major: u16, minor: u16, arch: Arch) -> Vec<String> {
let mut formats = vec![match arch {
Arch::Aarch64 => "arm64".to_string(),
_ => arch.to_string(),
}];
if matches!(arch, Arch::X86_64) {
if (major, minor) < (10, 4) {
return vec![];
}
formats.extend([
"intel".to_string(),
"fat64".to_string(),
"fat32".to_string(),
]);
}
if matches!(arch, Arch::X86_64 | Arch::Aarch64) {
formats.push("universal2".to_string());
}
if matches!(arch, Arch::X86_64) {
formats.push("universal".to_string());
}
formats
}

4
crates/puffin-resolver/Cargo.toml
View file

@ -10,9 +10,11 @@ authors.workspace = true
license.workspace = true
[dependencies]
platform-tags = { path = "../platform-tags" }
puffin-client = { path = "../puffin-client" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" }
platform-host = { path = "../platform-host" }
wheel-filename = { path = "../wheel-filename" }
anyhow = { workspace = true }
bitflags = { workspace = true }

8
crates/puffin-resolver/src/lib.rs
View file

@ -5,16 +5,16 @@ use anyhow::Result;
use bitflags::bitflags;
use futures::future::Either;
use futures::{StreamExt, TryFutureExt};
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use tracing::debug;
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use platform_tags::Tags;
use puffin_client::{File, PypiClient, SimpleJson};
use puffin_package::metadata::Metadata21;
use puffin_package::package_name::PackageName;
use puffin_package::requirements::Requirements;
use puffin_package::wheel::WheelFilename;
use puffin_platform::tags::Tags;
use wheel_filename::WheelFilename;
#[derive(Debug)]
pub struct Resolution(HashMap<PackageName, PinnedPackage>);

15
crates/wheel-filename/Cargo.toml Normal file
View file

@ -0,0 +1,15 @@
[package]
name = "wheel-filename"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
platform-tags = { path = "../platform-tags" }
thiserror = { workspace = true }

2
crates/puffin-package/src/wheel.rs → crates/wheel-filename/src/lib.rs
View file

@ -2,7 +2,7 @@ use std::str::FromStr;
use thiserror::Error;
use puffin_platform::tags::Tags;
use platform_tags::Tags;
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WheelFilename {