mirror of
https://github.com/astral-sh/uv.git
synced 2025-11-20 03:49:54 +00:00
Remove some unused code from install-wheel-rs (#2001)
I need to make a bunch of changes to this crate, and I'm finding that the existing unused interfaces are really getting in the way.
This commit is contained in:
Charlie Marsh
GitHub
parent
6678d545fb
commit
5997d0da3d
15 changed files with 45 additions and 826 deletions
|
|
@ -1,75 +1,5 @@
|
|||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use fs2::FileExt;
|
||||
use fs_err::File;
|
||||
use tracing::{error, warn};
|
||||
|
||||
use uv_fs::Normalized;
|
||||
|
||||
const INSTALL_LOCKFILE: &str = "install-wheel-rs.lock";
|
||||
|
||||
/// I'm not sure that's the right way to normalize here, but it's a single place to change
|
||||
/// everything.
|
||||
///
|
||||
/// For displaying to the user, `-` is better, and it's also what poetry lockfile 2.0 does
|
||||
///
|
||||
/// Keep in sync with `find_distributions`
|
||||
pub fn normalize_name(dep_name: &str) -> String {
|
||||
dep_name.to_lowercase().replace(['.', '_'], "-")
|
||||
}
|
||||
|
||||
/// A directory for which we acquired a install-wheel-rs.lock lockfile
|
||||
pub struct LockedDir {
|
||||
/// The directory to lock
|
||||
path: PathBuf,
|
||||
/// handle on the install-wheel-rs.lock that drops the lock
|
||||
lockfile: File,
|
||||
}
|
||||
|
||||
impl LockedDir {
|
||||
/// Tries to lock the directory, returns Ok(None) if it is already locked
|
||||
pub fn try_acquire(path: &Path) -> io::Result<Option<Self>> {
|
||||
let lockfile = File::create(path.join(INSTALL_LOCKFILE))?;
|
||||
if lockfile.file().try_lock_exclusive().is_ok() {
|
||||
Ok(Some(Self {
|
||||
path: path.to_path_buf(),
|
||||
lockfile,
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Locks the directory, if necessary blocking until the lock becomes free
|
||||
pub fn acquire(path: &Path) -> io::Result<Self> {
|
||||
let lockfile = File::create(path.join(INSTALL_LOCKFILE))?;
|
||||
lockfile.file().lock_exclusive()?;
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
lockfile,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockedDir {
|
||||
fn drop(&mut self) {
|
||||
if let Err(err) = self.lockfile.file().unlock() {
|
||||
error!(
|
||||
"Failed to unlock {}: {}",
|
||||
self.lockfile.path().normalized_display(),
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<Path> for LockedDir {
|
||||
fn as_ref(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
/// A virtual environment into which a wheel can be installed.
|
||||
///
|
||||
/// We use a lockfile to prevent multiple instance writing stuff on the same time
|
||||
|
|
@ -109,22 +39,3 @@ impl<T: AsRef<Path>> InstallLocation<T> {
|
|||
&self.venv_root
|
||||
}
|
||||
}
|
||||
|
||||
impl InstallLocation<PathBuf> {
|
||||
pub fn acquire_lock(&self) -> io::Result<InstallLocation<LockedDir>> {
|
||||
let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_root)? {
|
||||
locked_dir
|
||||
} else {
|
||||
warn!(
|
||||
"Could not acquire exclusive lock for installing, is another installation process \
|
||||
running? Sleeping until lock becomes free"
|
||||
);
|
||||
LockedDir::acquire(&self.venv_root)?
|
||||
};
|
||||
|
||||
Ok(InstallLocation {
|
||||
venv_root: locked_dir,
|
||||
python_version: self.python_version,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,22 +11,15 @@ use zip::result::ZipError;
|
|||
use zip::ZipArchive;
|
||||
|
||||
use distribution_filename::WheelFilename;
|
||||
pub use install_location::{normalize_name, InstallLocation, LockedDir};
|
||||
pub use install_location::InstallLocation;
|
||||
use pep440_rs::Version;
|
||||
use platform_host::{Arch, Os};
|
||||
pub use record::RecordEntry;
|
||||
pub use script::Script;
|
||||
pub use uninstall::{uninstall_wheel, Uninstall};
|
||||
use uv_fs::Normalized;
|
||||
use uv_normalize::PackageName;
|
||||
pub use wheel::{
|
||||
install_wheel, parse_key_value_file, read_record_file, relative_to, SHEBANG_PYTHON,
|
||||
};
|
||||
|
||||
mod install_location;
|
||||
pub mod linker;
|
||||
#[cfg(feature = "python_bindings")]
|
||||
mod python_bindings;
|
||||
mod record;
|
||||
mod script;
|
||||
mod uninstall;
|
||||
|
|
|
|||
|
|
@ -10,17 +10,18 @@ use reflink_copy as reflink;
|
|||
use tempfile::tempdir_in;
|
||||
use tracing::{debug, instrument};
|
||||
|
||||
use crate::Error;
|
||||
use distribution_filename::WheelFilename;
|
||||
use pep440_rs::Version;
|
||||
use pypi_types::DirectUrl;
|
||||
use uv_normalize::PackageName;
|
||||
|
||||
use crate::install_location::InstallLocation;
|
||||
use crate::script::scripts_from_ini;
|
||||
use crate::script::{scripts_from_ini, Script};
|
||||
use crate::wheel::{
|
||||
extra_dist_info, install_data, parse_metadata, parse_wheel_version, write_script_entrypoints,
|
||||
extra_dist_info, install_data, parse_metadata, parse_wheel_version, read_record_file,
|
||||
write_script_entrypoints,
|
||||
};
|
||||
use crate::{read_record_file, Error, Script};
|
||||
|
||||
/// Install the given wheel to the given venv
|
||||
///
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
|
||||
use clap::Parser;
|
||||
use fs_err::File;
|
||||
#[cfg(feature = "rayon")]
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
|
||||
use distribution_filename::WheelFilename;
|
||||
use install_wheel_rs::{install_wheel, Error, InstallLocation};
|
||||
|
||||
/// Low level install CLI, mainly used for testing
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
wheels: Vec<PathBuf>,
|
||||
/// The root of the venv to install in
|
||||
#[clap(long, env = "VIRTUAL_ENV")]
|
||||
venv: PathBuf,
|
||||
/// The major version of the current python interpreter
|
||||
#[clap(long)]
|
||||
major: u8,
|
||||
/// The minor version of the current python interpreter
|
||||
#[clap(long)]
|
||||
minor: u8,
|
||||
/// Compile .py files to .pyc (errors are ignored)
|
||||
#[clap(long)]
|
||||
compile: bool,
|
||||
/// Don't check the hashes in RECORD
|
||||
#[clap(long)]
|
||||
skip_hashes: bool,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Error> {
|
||||
let args = Args::parse();
|
||||
let venv_base = args.venv.canonicalize()?;
|
||||
let location = InstallLocation::new(venv_base, (args.major, args.minor));
|
||||
let locked_dir = location.acquire_lock()?;
|
||||
|
||||
let wheels: Vec<(PathBuf, WheelFilename)> = args
|
||||
.wheels
|
||||
.into_iter()
|
||||
.map(|wheel| {
|
||||
let filename = wheel
|
||||
.file_name()
|
||||
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
|
||||
.to_string_lossy();
|
||||
let filename = WheelFilename::from_str(&filename)?;
|
||||
Ok((wheel, filename))
|
||||
})
|
||||
.collect::<Result<_, Error>>()?;
|
||||
|
||||
let wheels = {
|
||||
#[cfg(feature = "rayon")]
|
||||
{
|
||||
wheels.into_par_iter()
|
||||
}
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
wheels.into_iter()
|
||||
}
|
||||
};
|
||||
wheels
|
||||
.map(|(wheel, filename)| {
|
||||
install_wheel(
|
||||
&locked_dir,
|
||||
File::open(wheel)?,
|
||||
&filename,
|
||||
None,
|
||||
None,
|
||||
args.compile,
|
||||
!args.skip_hashes,
|
||||
&[],
|
||||
location.python(),
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.collect::<Result<_, Error>>()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
#![allow(clippy::format_push_string)] // I will not replace clear and infallible with fallible, io looking code
|
||||
|
||||
use crate::{install_wheel, Error, InstallLocation, LockedDir};
|
||||
use distribution_filename::WheelFilename;
|
||||
use fs_err::File;
|
||||
use pyo3::create_exception;
|
||||
use pyo3::types::PyModule;
|
||||
use pyo3::{pyclass, pymethods, pymodule, PyErr, PyResult, Python};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
|
||||
create_exception!(
|
||||
install_wheel_rs,
|
||||
PyWheelInstallerError,
|
||||
pyo3::exceptions::PyException
|
||||
);
|
||||
|
||||
impl From<Error> for PyErr {
|
||||
fn from(err: Error) -> Self {
|
||||
let mut accumulator = format!("Failed to install wheels: {err}");
|
||||
|
||||
let mut current_err: &dyn std::error::Error = &err;
|
||||
while let Some(cause) = current_err.source() {
|
||||
accumulator.push_str(&format!("\n Caused by: {cause}"));
|
||||
current_err = cause;
|
||||
}
|
||||
PyWheelInstallerError::new_err(accumulator)
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
struct LockedVenv {
|
||||
location: InstallLocation<LockedDir>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl LockedVenv {
|
||||
#[new]
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult<Self> {
|
||||
Ok(Self {
|
||||
location: InstallLocation::new(
|
||||
LockedDir::acquire(&venv)?,
|
||||
(py.version_info().major, py.version_info().minor),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn install_wheel(&self, py: Python, wheel: PathBuf) -> PyResult<()> {
|
||||
// Would be nicer through https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath
|
||||
let sys_executable: String = py.import("sys")?.getattr("executable")?.extract()?;
|
||||
|
||||
// TODO: Pass those options on to the user
|
||||
py.allow_threads(|| {
|
||||
let filename = wheel
|
||||
.file_name()
|
||||
.ok_or_else(|| Error::InvalidWheel("Expected a file".to_string()))?
|
||||
.to_string_lossy();
|
||||
let filename = WheelFilename::from_str(&filename)?;
|
||||
install_wheel(
|
||||
&self.location,
|
||||
File::open(wheel)?,
|
||||
&filename,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
true,
|
||||
&[],
|
||||
Path::new(&sys_executable),
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[pymodule]
|
||||
pub(crate) fn install_wheel_rs(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
// Good enough for now
|
||||
if env::var_os("RUST_LOG").is_some() {
|
||||
tracing_subscriber::fmt::init();
|
||||
} else {
|
||||
let format = tracing_subscriber::fmt::format()
|
||||
.with_level(false)
|
||||
.with_target(false)
|
||||
.without_time()
|
||||
.compact();
|
||||
tracing_subscriber::fmt().event_format(format).init();
|
||||
}
|
||||
m.add_class::<LockedVenv>()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -8,9 +8,9 @@ use serde::{Deserialize, Serialize};
|
|||
/// tqdm-4.62.3.dist-info/RECORD,,
|
||||
/// ```
|
||||
#[derive(Deserialize, Serialize, PartialOrd, PartialEq, Ord, Eq)]
|
||||
pub struct RecordEntry {
|
||||
pub path: String,
|
||||
pub hash: Option<String>,
|
||||
pub(crate) struct RecordEntry {
|
||||
pub(crate) path: String,
|
||||
pub(crate) hash: Option<String>,
|
||||
#[allow(dead_code)]
|
||||
pub size: Option<u64>,
|
||||
pub(crate) size: Option<u64>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,26 +8,11 @@ use crate::{wheel, Error};
|
|||
|
||||
/// A script defining the name of the runnable entrypoint and the module and function that should be
|
||||
/// run.
|
||||
#[cfg(feature = "python_bindings")]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
|
||||
#[pyo3::pyclass(dict)]
|
||||
pub struct Script {
|
||||
#[pyo3(get)]
|
||||
pub script_name: String,
|
||||
#[pyo3(get)]
|
||||
pub module: String,
|
||||
#[pyo3(get)]
|
||||
pub function: String,
|
||||
}
|
||||
|
||||
/// A script defining the name of the runnable entrypoint and the module and function that should be
|
||||
/// run.
|
||||
#[cfg(not(feature = "python_bindings"))]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
|
||||
pub struct Script {
|
||||
pub script_name: String,
|
||||
pub module: String,
|
||||
pub function: String,
|
||||
pub(crate) struct Script {
|
||||
pub(crate) name: String,
|
||||
pub(crate) module: String,
|
||||
pub(crate) function: String,
|
||||
}
|
||||
|
||||
impl Script {
|
||||
|
|
@ -36,7 +21,7 @@ impl Script {
|
|||
/// <https://packaging.python.org/en/latest/specifications/entry-points/>
|
||||
///
|
||||
/// Extras are supposed to be ignored, which happens if you pass None for extras
|
||||
pub fn from_value(
|
||||
pub(crate) fn from_value(
|
||||
script_name: &str,
|
||||
value: &str,
|
||||
extras: Option<&[String]>,
|
||||
|
|
@ -66,13 +51,13 @@ impl Script {
|
|||
}
|
||||
|
||||
Ok(Some(Self {
|
||||
script_name: script_name.to_string(),
|
||||
name: script_name.to_string(),
|
||||
module: captures.name("module").unwrap().as_str().to_string(),
|
||||
function: captures.name("function").unwrap().as_str().to_string(),
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn import_name(&self) -> &str {
|
||||
pub(crate) fn import_name(&self) -> &str {
|
||||
self.function
|
||||
.split_once('.')
|
||||
.map_or(&self.function, |(import_name, _)| import_name)
|
||||
|
|
@ -104,13 +89,13 @@ pub(crate) fn scripts_from_ini(
|
|||
// https://github.com/pypa/pip/blob/3898741e29b7279e7bffe044ecfbe20f6a438b1e/src/pip/_internal/operations/install/wheel.py#L283
|
||||
// https://github.com/astral-sh/uv/issues/1593
|
||||
for script in &mut console_scripts {
|
||||
let Some((left, right)) = script.script_name.split_once('.') else {
|
||||
let Some((left, right)) = script.name.split_once('.') else {
|
||||
continue;
|
||||
};
|
||||
if left != "pip3" || right.parse::<u8>().is_err() {
|
||||
continue;
|
||||
}
|
||||
script.script_name = format!("pip3.{python_minor}");
|
||||
script.name = format!("pip3.{python_minor}");
|
||||
}
|
||||
|
||||
Ok((console_scripts, gui_scripts))
|
||||
|
|
@ -118,7 +103,7 @@ pub(crate) fn scripts_from_ini(
|
|||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use crate::Script;
|
||||
use crate::script::Script;
|
||||
|
||||
#[test]
|
||||
fn test_valid_script_names() {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ use std::path::{Component, Path, PathBuf};
|
|||
use fs_err as fs;
|
||||
use tracing::debug;
|
||||
|
||||
use crate::{read_record_file, Error};
|
||||
use crate::wheel::read_record_file;
|
||||
use crate::Error;
|
||||
|
||||
/// Uninstall the wheel represented by the given `dist_info` directory.
|
||||
pub fn uninstall_wheel(dist_info: &Path) -> Result<Uninstall, Error> {
|
||||
|
|
|
|||
|
|
@ -1,36 +1,27 @@
|
|||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
|
||||
use std::io::{BufRead, BufReader, Cursor, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitStatus, Stdio};
|
||||
use std::str::FromStr;
|
||||
use std::{env, io, iter};
|
||||
|
||||
use data_encoding::BASE64URL_NOPAD;
|
||||
use fs_err as fs;
|
||||
use fs_err::{DirEntry, File};
|
||||
use mailparse::MailHeaderMap;
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use rustc_hash::FxHashMap;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tempfile::tempdir;
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use walkdir::WalkDir;
|
||||
use zip::result::ZipError;
|
||||
use zip::write::FileOptions;
|
||||
use zip::{ZipArchive, ZipWriter};
|
||||
|
||||
use distribution_filename::WheelFilename;
|
||||
use pep440_rs::Version;
|
||||
use tracing::{instrument, warn};
|
||||
use walkdir::WalkDir;
|
||||
use zip::write::FileOptions;
|
||||
use zip::ZipWriter;
|
||||
|
||||
use pypi_types::DirectUrl;
|
||||
use uv_fs::Normalized;
|
||||
use uv_normalize::PackageName;
|
||||
|
||||
use crate::install_location::{InstallLocation, LockedDir};
|
||||
use crate::install_location::InstallLocation;
|
||||
use crate::record::RecordEntry;
|
||||
use crate::script::{scripts_from_ini, Script};
|
||||
use crate::{find_dist_info, Error};
|
||||
|
||||
/// `#!/usr/bin/env python`
|
||||
pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python";
|
||||
use crate::script::Script;
|
||||
use crate::Error;
|
||||
|
||||
const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V'];
|
||||
|
||||
|
|
@ -97,27 +88,6 @@ pub(crate) fn read_scripts_from_section(
|
|||
Ok(scripts)
|
||||
}
|
||||
|
||||
/// Parses the `entry_points.txt` entry in the wheel for console scripts
|
||||
///
|
||||
/// Returns (`script_name`, module, function)
|
||||
///
|
||||
/// Extras are supposed to be ignored, which happens if you pass None for extras
|
||||
fn parse_scripts<R: Read + Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
dist_info_dir: &str,
|
||||
extras: Option<&[String]>,
|
||||
python_minor: u8,
|
||||
) -> Result<(Vec<Script>, Vec<Script>), Error> {
|
||||
let entry_points_path = format!("{dist_info_dir}/entry_points.txt");
|
||||
let ini = match archive.by_name(&entry_points_path) {
|
||||
Ok(file) => std::io::read_to_string(file)?,
|
||||
Err(ZipError::FileNotFound) => return Ok((Vec::new(), Vec::new())),
|
||||
Err(err) => return Err(Error::Zip(entry_points_path, err)),
|
||||
};
|
||||
|
||||
scripts_from_ini(extras, python_minor, ini)
|
||||
}
|
||||
|
||||
/// Shamelessly stolen (and updated for recent sha2)
|
||||
/// <https://github.com/richo/hashing-copy/blob/d8dd2fdb63c6faf198de0c9e5713d6249cbb5323/src/lib.rs#L10-L52>
|
||||
/// which in turn got it from std
|
||||
|
|
@ -146,125 +116,6 @@ fn copy_and_hash(reader: &mut impl Read, writer: &mut impl Write) -> io::Result<
|
|||
))
|
||||
}
|
||||
|
||||
/// Extract all files from the wheel into the site packages
|
||||
///
|
||||
/// Matches with the RECORD entries
|
||||
///
|
||||
/// Returns paths relative to site packages
|
||||
fn unpack_wheel_files<R: Read + Seek>(
|
||||
site_packages: &Path,
|
||||
record_path: &str,
|
||||
archive: &mut ZipArchive<R>,
|
||||
record: &[RecordEntry],
|
||||
check_hashes: bool,
|
||||
) -> Result<Vec<PathBuf>, Error> {
|
||||
let mut extracted_paths = Vec::new();
|
||||
// Cache the created parent dirs to avoid io calls
|
||||
// When deactivating bytecode compilation and sha2 those were 5% of total runtime, with
|
||||
// cache it 2.3%
|
||||
let mut created_dirs = FxHashSet::default();
|
||||
// https://github.com/zip-rs/zip/blob/7edf2489d5cff8b80f02ee6fc5febf3efd0a9442/examples/extract.rs
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i).map_err(|err| {
|
||||
let file1 = format!("(index {i})");
|
||||
Error::Zip(file1, err)
|
||||
})?;
|
||||
// enclosed_name takes care of evil zip paths
|
||||
let relative = match file.enclosed_name() {
|
||||
Some(path) => path.to_owned(),
|
||||
None => continue,
|
||||
};
|
||||
let out_path = site_packages.join(&relative);
|
||||
|
||||
if file.name().ends_with('/') {
|
||||
// pip seems to do ignore those folders, so do we
|
||||
// fs::create_dir_all(&out_path)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(parent) = out_path.parent() {
|
||||
if created_dirs.insert(parent.to_path_buf()) {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
}
|
||||
let mut outfile = BufWriter::new(File::create(&out_path)?);
|
||||
let encoded_hash = if check_hashes {
|
||||
let (_size, encoded_hash) = copy_and_hash(&mut file, &mut outfile)?;
|
||||
Some(encoded_hash)
|
||||
} else {
|
||||
io::copy(&mut file, &mut outfile)?;
|
||||
None
|
||||
};
|
||||
|
||||
extracted_paths.push(relative.clone());
|
||||
|
||||
// Get and Set permissions
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::fs::Permissions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
if let Some(mode) = file.unix_mode() {
|
||||
fs::set_permissions(&out_path, Permissions::from_mode(mode))?;
|
||||
}
|
||||
}
|
||||
|
||||
// This is the RECORD file that contains the hashes so naturally it can't contain it's own
|
||||
// hash and size (but it does contain an entry with two empty fields)
|
||||
// > 6. RECORD.jws is used for digital signatures. It is not mentioned in RECORD.
|
||||
// > 7. RECORD.p7s is allowed as a courtesy to anyone who would prefer to use S/MIME
|
||||
// > signatures to secure their wheel files. It is not mentioned in RECORD.
|
||||
let record_path = PathBuf::from(&record_path);
|
||||
if [
|
||||
record_path.clone(),
|
||||
record_path.with_extension("jws"),
|
||||
record_path.with_extension("p7s"),
|
||||
]
|
||||
.contains(&relative)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(encoded_hash) = encoded_hash {
|
||||
// `relative == Path::new(entry.path)` was really slow
|
||||
let relative_str = relative.display().to_string();
|
||||
let recorded_hash = record
|
||||
.iter()
|
||||
.find(|entry| relative_str == entry.path)
|
||||
.and_then(|entry| entry.hash.as_ref())
|
||||
.ok_or_else(|| {
|
||||
Error::RecordFile(format!(
|
||||
"Missing hash for {} (expected {})",
|
||||
relative.normalized_display(),
|
||||
encoded_hash
|
||||
))
|
||||
})?;
|
||||
if recorded_hash != &encoded_hash {
|
||||
if relative.as_os_str().to_string_lossy().starts_with("torch-") {
|
||||
error!(
|
||||
"Hash mismatch for {}. Recorded: {}, Actual: {}",
|
||||
relative.normalized_display(),
|
||||
recorded_hash,
|
||||
encoded_hash,
|
||||
);
|
||||
error!(
|
||||
"Torch isn't capable of producing correct hashes 🙄 Ignoring. \
|
||||
https://github.com/pytorch/pytorch/issues/47916"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return Err(Error::RecordFile(format!(
|
||||
"Hash mismatch for {}. Recorded: {}, Actual: {}",
|
||||
relative.normalized_display(),
|
||||
recorded_hash,
|
||||
encoded_hash,
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(extracted_paths)
|
||||
}
|
||||
|
||||
fn get_shebang(location: &InstallLocation<impl AsRef<Path>>) -> String {
|
||||
format!("#!{}", location.python().normalized().display())
|
||||
}
|
||||
|
|
@ -359,15 +210,15 @@ pub(crate) fn write_script_entrypoints(
|
|||
let entrypoint_relative = if cfg!(windows) {
|
||||
// On windows we actually build an .exe wrapper
|
||||
let script_name = entrypoint
|
||||
.script_name
|
||||
.name
|
||||
// FIXME: What are the in-reality rules here for names?
|
||||
.strip_suffix(".py")
|
||||
.unwrap_or(&entrypoint.script_name)
|
||||
.unwrap_or(&entrypoint.name)
|
||||
.to_string()
|
||||
+ ".exe";
|
||||
bin_rel().join(script_name)
|
||||
} else {
|
||||
bin_rel().join(&entrypoint.script_name)
|
||||
bin_rel().join(&entrypoint.name)
|
||||
};
|
||||
|
||||
// Generate the launcher script.
|
||||
|
|
@ -456,156 +307,12 @@ pub(crate) fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Call `python -m compileall` to generate pyc file for the installed code
|
||||
///
|
||||
/// 2.f Compile any installed .py to .pyc. (Uninstallers should be smart enough to remove .pyc
|
||||
/// even if it is not mentioned in RECORD.)
|
||||
#[instrument(skip_all)]
|
||||
fn bytecode_compile(
|
||||
site_packages: &Path,
|
||||
unpacked_paths: Vec<PathBuf>,
|
||||
python_version: (u8, u8),
|
||||
sys_executable: &Path,
|
||||
// Only for logging
|
||||
name: &str,
|
||||
record: &mut Vec<RecordEntry>,
|
||||
) -> Result<(), Error> {
|
||||
// https://github.com/pypa/pip/blob/b5457dfee47dd9e9f6ec45159d9d410ba44e5ea1/src/pip/_internal/operations/install/wheel.py#L592-L603
|
||||
let py_source_paths: Vec<_> = unpacked_paths
|
||||
.into_iter()
|
||||
.filter(|path| {
|
||||
path.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("py"))
|
||||
&& site_packages.join(path).is_file()
|
||||
})
|
||||
.collect();
|
||||
|
||||
// bytecode compiling crashes non-deterministically with various errors, from syntax errors
|
||||
// to cpython segmentation faults, so we add a simple retry loop
|
||||
let mut retries = 3;
|
||||
let (status, lines) = loop {
|
||||
let (status, lines) =
|
||||
bytecode_compile_inner(site_packages, &py_source_paths, sys_executable)?;
|
||||
retries -= 1;
|
||||
if status.success() || retries == 0 {
|
||||
break (status, lines);
|
||||
}
|
||||
|
||||
warn!("Failed to compile {name} with python compileall, retrying",);
|
||||
};
|
||||
if !status.success() {
|
||||
// lossy because we want the error reporting to survive c̴̞̏ü̸̜̹̈́ŕ̴͉̈ś̷̤ė̵̤͋d̷͙̄ filenames in the zip
|
||||
return Err(Error::PythonSubcommand(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("Failed to run python compileall, log above: {status}"),
|
||||
)));
|
||||
}
|
||||
|
||||
// like pip, we just ignored all that failed to compile
|
||||
// Add each that succeeded to the RECORD
|
||||
for py_path in lines {
|
||||
let py_path = py_path.trim();
|
||||
if py_path.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let py_path = Path::new(py_path);
|
||||
let pyc_path = py_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new(""))
|
||||
.join("__pycache__")
|
||||
// Unwrap is save because we checked for an extension before
|
||||
.join(py_path.file_name().unwrap())
|
||||
.with_extension(format!(
|
||||
"cpython-{}{}.pyc",
|
||||
python_version.0, python_version.1
|
||||
));
|
||||
if !site_packages.join(&pyc_path).is_file() {
|
||||
return Err(Error::PythonSubcommand(io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
format!(
|
||||
"Didn't find pyc generated by compileall: {}",
|
||||
site_packages.join(&pyc_path).normalized_display()
|
||||
),
|
||||
)));
|
||||
}
|
||||
// 2.d Update distribution-1.0.dist-info/RECORD with the installed paths.
|
||||
|
||||
// https://www.python.org/dev/peps/pep-0376/#record
|
||||
// > [..] a hash of the file's contents. Notice that pyc and pyo generated files don't have
|
||||
// > any hash because they are automatically produced from py files. So checking the hash of
|
||||
// > the corresponding py file is enough to decide if the file and its associated pyc or pyo
|
||||
// > files have changed.
|
||||
record.push(RecordEntry {
|
||||
path: pyc_path.display().to_string(),
|
||||
hash: None,
|
||||
size: None,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The actual command part which we repeat if it fails
|
||||
fn bytecode_compile_inner(
|
||||
site_packages: &Path,
|
||||
py_source_paths: &[PathBuf],
|
||||
sys_executable: &Path,
|
||||
) -> Result<(ExitStatus, Vec<String>), Error> {
|
||||
let temp_dir = tempdir()?;
|
||||
// Running python with an actual file will produce better error messages
|
||||
let pip_compileall_py = temp_dir.path().join("pip_compileall.py");
|
||||
fs::write(&pip_compileall_py, include_str!("pip_compileall.py"))?;
|
||||
// We input the paths through stdin and get the successful paths returned through stdout
|
||||
let mut bytecode_compiler = Command::new(sys_executable)
|
||||
.arg(&pip_compileall_py)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.current_dir(site_packages.normalized())
|
||||
.spawn()
|
||||
.map_err(Error::PythonSubcommand)?;
|
||||
|
||||
// https://stackoverflow.com/questions/49218599/write-to-child-process-stdin-in-rust/49597789#comment120223107_49597789
|
||||
let mut child_stdin = bytecode_compiler
|
||||
.stdin
|
||||
.take()
|
||||
.expect("Child must have stdin");
|
||||
|
||||
// Pass paths newline terminated to compileall
|
||||
for path in py_source_paths {
|
||||
debug!("bytecode compiling {}", path.display());
|
||||
// There is no OsStr -> Bytes conversion on windows :o
|
||||
// https://stackoverflow.com/questions/43083544/how-can-i-convert-osstr-to-u8-vecu8-on-windows
|
||||
writeln!(&mut child_stdin, "{}", path.display()).map_err(Error::PythonSubcommand)?;
|
||||
}
|
||||
// Close stdin to finish and avoid indefinite blocking
|
||||
drop(child_stdin);
|
||||
|
||||
// Already read stdout here to avoid it running full (pipes are limited)
|
||||
let stdout = bytecode_compiler.stdout.take().unwrap();
|
||||
let mut lines: Vec<String> = Vec::new();
|
||||
for line in BufReader::new(stdout).lines() {
|
||||
let line = line.map_err(|err| {
|
||||
Error::PythonSubcommand(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("Invalid utf-8 returned by python compileall: {err}"),
|
||||
))
|
||||
})?;
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
let output = bytecode_compiler
|
||||
.wait_with_output()
|
||||
.map_err(Error::PythonSubcommand)?;
|
||||
Ok((output.status, lines))
|
||||
}
|
||||
|
||||
/// Give the path relative to the base directory
|
||||
///
|
||||
/// lib/python/site-packages/foo/__init__.py and lib/python/site-packages -> foo/__init__.py
|
||||
/// lib/marker.txt and lib/python/site-packages -> ../../marker.txt
|
||||
/// `bin/foo_launcher` and lib/python/site-packages -> ../../../`bin/foo_launcher`
|
||||
pub fn relative_to(path: &Path, base: &Path) -> Result<PathBuf, Error> {
|
||||
pub(crate) fn relative_to(path: &Path, base: &Path) -> Result<PathBuf, Error> {
|
||||
// Find the longest common prefix, and also return the path stripped from that prefix
|
||||
let (stripped, common_prefix) = base
|
||||
.ancestors()
|
||||
|
|
@ -787,7 +494,7 @@ pub(crate) fn install_data(
|
|||
if console_scripts
|
||||
.iter()
|
||||
.chain(gui_scripts)
|
||||
.any(|script| script.script_name == match_name)
|
||||
.any(|script| script.name == match_name)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
|
@ -884,7 +591,7 @@ pub(crate) fn extra_dist_info(
|
|||
|
||||
/// Reads the record file
|
||||
/// <https://www.python.org/dev/peps/pep-0376/#record>
|
||||
pub fn read_record_file(record: &mut impl Read) -> Result<Vec<RecordEntry>, Error> {
|
||||
pub(crate) fn read_record_file(record: &mut impl Read) -> Result<Vec<RecordEntry>, Error> {
|
||||
csv::ReaderBuilder::new()
|
||||
.has_headers(false)
|
||||
.escape(Some(b'"'))
|
||||
|
|
@ -902,7 +609,7 @@ pub fn read_record_file(record: &mut impl Read) -> Result<Vec<RecordEntry>, Erro
|
|||
}
|
||||
|
||||
/// Parse a file with `Key: value` entries such as WHEEL and METADATA
|
||||
pub fn parse_key_value_file(
|
||||
fn parse_key_value_file(
|
||||
file: &mut impl Read,
|
||||
debug_filename: &str,
|
||||
) -> Result<FxHashMap<String, Vec<String>>, Error> {
|
||||
|
|
@ -926,197 +633,6 @@ pub fn parse_key_value_file(
|
|||
Ok(data)
|
||||
}
|
||||
|
||||
/// Install the given wheel to the given venv
|
||||
///
|
||||
/// The caller must ensure that the wheel is compatible to the environment.
|
||||
///
|
||||
/// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl>
|
||||
///
|
||||
/// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/>
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[instrument(skip_all, fields(name = % filename.name))]
|
||||
pub fn install_wheel(
|
||||
location: &InstallLocation<LockedDir>,
|
||||
reader: impl Read + Seek,
|
||||
filename: &WheelFilename,
|
||||
direct_url: Option<&DirectUrl>,
|
||||
installer: Option<&str>,
|
||||
compile: bool,
|
||||
check_hashes: bool,
|
||||
// initially used to the console scripts, currently unused. Keeping it because we likely need
|
||||
// it for validation later.
|
||||
_extras: &[String],
|
||||
sys_executable: impl AsRef<Path>,
|
||||
) -> Result<String, Error> {
|
||||
let name = &filename.name;
|
||||
|
||||
let base_location = location.venv_root();
|
||||
|
||||
let site_packages_python = format!(
|
||||
"python{}.{}",
|
||||
location.python_version().0,
|
||||
location.python_version().1
|
||||
);
|
||||
let site_packages = if cfg!(target_os = "windows") {
|
||||
base_location.as_ref().join("Lib").join("site-packages")
|
||||
} else {
|
||||
base_location
|
||||
.as_ref()
|
||||
.join("lib")
|
||||
.join(site_packages_python)
|
||||
.join("site-packages")
|
||||
};
|
||||
|
||||
debug!(name = name.as_ref(), "Opening zip");
|
||||
// No BufReader: https://github.com/zip-rs/zip/issues/381
|
||||
let mut archive = ZipArchive::new(reader).map_err(|err| {
|
||||
let file = "(index)".to_string();
|
||||
Error::Zip(file, err)
|
||||
})?;
|
||||
|
||||
debug!(name = name.as_ref(), "Getting wheel metadata");
|
||||
let dist_info_prefix = find_dist_info(filename, archive.file_names().map(|name| (name, name)))?
|
||||
.1
|
||||
.to_string();
|
||||
let metadata = dist_info_metadata(&dist_info_prefix, &mut archive)?;
|
||||
let (name, version) = parse_metadata(&dist_info_prefix, &metadata)?;
|
||||
|
||||
// Validate the wheel name and version.
|
||||
{
|
||||
let name = PackageName::from_str(&name)?;
|
||||
if name != filename.name {
|
||||
return Err(Error::MismatchedName(name, filename.name.clone()));
|
||||
}
|
||||
|
||||
let version = Version::from_str(&version)?;
|
||||
if version != filename.version {
|
||||
return Err(Error::MismatchedVersion(version, filename.version.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let record_path = format!("{dist_info_prefix}.dist-info/RECORD");
|
||||
let mut record = read_record_file(&mut archive.by_name(&record_path).map_err(|err| {
|
||||
let file = record_path.clone();
|
||||
Error::Zip(file, err)
|
||||
})?)?;
|
||||
|
||||
// We're going step by step though
|
||||
// https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl
|
||||
// > 1.a Parse distribution-1.0.dist-info/WHEEL.
|
||||
// > 1.b Check that installer is compatible with Wheel-Version. Warn if minor version is greater, abort if major version is greater.
|
||||
let wheel_file_path = format!("{dist_info_prefix}.dist-info/WHEEL");
|
||||
let wheel_file = archive
|
||||
.by_name(&wheel_file_path)
|
||||
.map_err(|err| Error::Zip(wheel_file_path, err))?;
|
||||
let wheel_text = io::read_to_string(wheel_file)?;
|
||||
parse_wheel_version(&wheel_text)?;
|
||||
// > 1.c If Root-Is-Purelib == ‘true’, unpack archive into purelib (site-packages).
|
||||
// > 1.d Else unpack archive into platlib (site-packages).
|
||||
// We always install in the same virtualenv site packages
|
||||
debug!(name = name.as_str(), "Extracting file");
|
||||
let unpacked_paths = unpack_wheel_files(
|
||||
&site_packages,
|
||||
&record_path,
|
||||
&mut archive,
|
||||
&record,
|
||||
check_hashes,
|
||||
)?;
|
||||
debug!(
|
||||
name = name.as_str(),
|
||||
"Extracted {} files",
|
||||
unpacked_paths.len()
|
||||
);
|
||||
|
||||
debug!(name = name.as_str(), "Writing entrypoints");
|
||||
let (console_scripts, gui_scripts) = parse_scripts(
|
||||
&mut archive,
|
||||
&dist_info_prefix,
|
||||
None,
|
||||
location.python_version().1,
|
||||
)?;
|
||||
write_script_entrypoints(
|
||||
&site_packages,
|
||||
location,
|
||||
&console_scripts,
|
||||
&mut record,
|
||||
false,
|
||||
)?;
|
||||
write_script_entrypoints(&site_packages, location, &gui_scripts, &mut record, true)?;
|
||||
|
||||
let data_dir = site_packages.join(format!("{dist_info_prefix}.data"));
|
||||
// 2.a Unpacked archive includes distribution-1.0.dist-info/ and (if there is data) distribution-1.0.data/.
|
||||
// 2.b Move each subtree of distribution-1.0.data/ onto its destination path. Each subdirectory of distribution-1.0.data/ is a key into a dict of destination directories, such as distribution-1.0.data/(purelib|platlib|headers|scripts|data). The initially supported paths are taken from distutils.command.install.
|
||||
if data_dir.is_dir() {
|
||||
debug!(name = name.as_str(), "Installing data");
|
||||
install_data(
|
||||
base_location.as_ref(),
|
||||
&site_packages,
|
||||
&data_dir,
|
||||
&name,
|
||||
location,
|
||||
&console_scripts,
|
||||
&gui_scripts,
|
||||
&mut record,
|
||||
)?;
|
||||
// 2.c If applicable, update scripts starting with #!python to point to the correct interpreter.
|
||||
// Script are unsupported through data
|
||||
// 2.e Remove empty distribution-1.0.data directory.
|
||||
fs::remove_dir_all(data_dir)?;
|
||||
} else {
|
||||
debug!(name = name.as_str(), "No data");
|
||||
}
|
||||
|
||||
// 2.f Compile any installed .py to .pyc. (Uninstallers should be smart enough to remove .pyc even if it is not mentioned in RECORD.)
|
||||
if compile {
|
||||
debug!(name = name.as_str(), "Bytecode compiling");
|
||||
bytecode_compile(
|
||||
&site_packages,
|
||||
unpacked_paths,
|
||||
location.python_version(),
|
||||
sys_executable.as_ref(),
|
||||
name.as_str(),
|
||||
&mut record,
|
||||
)?;
|
||||
}
|
||||
|
||||
debug!(name = name.as_str(), "Writing extra metadata");
|
||||
|
||||
extra_dist_info(
|
||||
&site_packages,
|
||||
&dist_info_prefix,
|
||||
true,
|
||||
direct_url,
|
||||
installer,
|
||||
&mut record,
|
||||
)?;
|
||||
|
||||
debug!(name = name.as_str(), "Writing record");
|
||||
let mut record_writer = csv::WriterBuilder::new()
|
||||
.has_headers(false)
|
||||
.escape(b'"')
|
||||
.from_path(site_packages.join(record_path))?;
|
||||
record.sort();
|
||||
for entry in record {
|
||||
record_writer.serialize(entry)?;
|
||||
}
|
||||
|
||||
Ok(filename.get_tag())
|
||||
}
|
||||
|
||||
/// Read the `dist-info` metadata from a wheel archive.
|
||||
fn dist_info_metadata(
|
||||
dist_info_prefix: &str,
|
||||
archive: &mut ZipArchive<impl Read + Seek + Sized>,
|
||||
) -> Result<Vec<u8>, Error> {
|
||||
let mut content = Vec::new();
|
||||
let dist_info_file = format!("{dist_info_prefix}.dist-info/METADATA");
|
||||
archive
|
||||
.by_name(&dist_info_file)
|
||||
.map_err(|err| Error::Zip(dist_info_file.clone(), err))?
|
||||
.read_to_end(&mut content)?;
|
||||
Ok(content)
|
||||
}
|
||||
|
||||
/// Parse the distribution name and version from a wheel's `dist-info` metadata.
|
||||
///
|
||||
/// See: <https://github.com/PyO3/python-pkginfo-rs>
|
||||
|
|
@ -1256,7 +772,7 @@ mod test {
|
|||
assert_eq!(
|
||||
Script::from_value("launcher", "foo.bar:main", None).unwrap(),
|
||||
Some(Script {
|
||||
script_name: "launcher".to_string(),
|
||||
name: "launcher".to_string(),
|
||||
module: "foo.bar".to_string(),
|
||||
function: "main".to_string(),
|
||||
})
|
||||
|
|
@ -1269,7 +785,7 @@ mod test {
|
|||
)
|
||||
.unwrap(),
|
||||
Some(Script {
|
||||
script_name: "launcher".to_string(),
|
||||
name: "launcher".to_string(),
|
||||
module: "foo.bar".to_string(),
|
||||
function: "main".to_string(),
|
||||
})
|
||||
|
|
@ -1286,7 +802,7 @@ mod test {
|
|||
)
|
||||
.unwrap(),
|
||||
Some(Script {
|
||||
script_name: "launcher".to_string(),
|
||||
name: "launcher".to_string(),
|
||||
module: "foomod".to_string(),
|
||||
function: "main_bar".to_string(),
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue