mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 21:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Remove Monotrail-specific code from install-wheel-rs (#68)

Browse source 
I think this isn't necessary to support in this generic crate. If we
choose to adopt Monotrail-style concepts, we'll likely need to rework
them anyway.
This commit is contained in:
Charlie Marsh 2023-10-08 18:28:57 -04:00 • committed by GitHub
parent adbee4fb32
commit 5b71cfdd0b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 78 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

7
crates/gourgeist/src/packages.rs
View file

@ -51,10 +51,7 @@ pub(crate) fn install_base_packages(
info: &InterpreterInfo, info: &InterpreterInfo,
paths: &VenvPaths, paths: &VenvPaths,
) -> Result<(), Error> { ) -> Result<(), Error> {
let install_location = InstallLocation::Venv { let install_location = InstallLocation::new(location.canonicalize()?, (info.major, info.minor));
venv_base: location.canonicalize()?,
python_version: (info.major, info.minor),
};
let install_location = install_location.acquire_lock()?; let install_location = install_location.acquire_lock()?;
// TODO: Use the json api instead // TODO: Use the json api instead
@ -79,8 +76,6 @@ pub(crate) fn install_base_packages(
false, false,
false, false,
&[], &[],
// Only relevant for monotrail style installation
"",
paths.interpreter.as_std_path(), paths.interpreter.as_std_path(),
) )
.map_err(|err| Error::InstallWheel { .map_err(|err| Error::InstallWheel {

128
crates/install-wheel-rs/src/install_location.rs
View file

@ -1,11 +1,8 @@
//! Multiplexing between venv install and monotrail install use std::io;
use std::path::{Path, PathBuf};
use fs2::FileExt; use fs2::FileExt;
use fs_err as fs;
use fs_err::File; use fs_err::File;
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use tracing::{error, warn}; use tracing::{error, warn};
const INSTALL_LOCKFILE: &str = "install-wheel-rs.lock"; const INSTALL_LOCKFILE: &str = "install-wheel-rs.lock";
@ -65,124 +62,65 @@ impl Drop for LockedDir {
} }
} }
impl Deref for LockedDir { impl AsRef<Path> for LockedDir {
type Target = Path; fn as_ref(&self) -> &Path {
fn deref(&self) -> &Self::Target {
&self.path &self.path
} }
} }
/// Multiplexing between venv install and monotrail install /// A virtual environment into which a wheel can be installed.
///
/// For monotrail, we have a structure that is {monotrail}/{normalized(name)}/{version}/tag
/// ///
/// We use a lockfile to prevent multiple instance writing stuff on the same time /// We use a lockfile to prevent multiple instance writing stuff on the same time
/// As of pip 22.0, e.g. `pip install numpy; pip install numpy; pip install numpy` will /// As of pip 22.0, e.g. `pip install numpy; pip install numpy; pip install numpy` will
/// nondeterministically fail /// non-deterministically fail.
/// pub struct InstallLocation<T: AsRef<Path>> {
/// I was also thinking about making a shared lock on the import side, but monotrail install /// absolute path
/// is supposedly atomic (by directory renaming), while for venv installation there can't be venv_base: T,
/// atomicity (we need to add lots of different file without a top level directory / key-turn python_version: (u8, u8),
/// file we could rename) and the locking would also need to happen in the import mechanism
/// itself to ensure
pub enum InstallLocation<T: Deref<Target = Path>> {
Venv {
/// absolute path
venv_base: T,
python_version: (u8, u8),
},
Monotrail {
monotrail_root: T,
python: PathBuf,
python_version: (u8, u8),
},
} }
impl<T: Deref<Target = Path>> InstallLocation<T> { impl<T: AsRef<Path>> InstallLocation<T> {
/// Returns the location of the python interpreter pub fn new(venv_base: T, python_version: (u8, u8)) -> Self {
pub fn get_python(&self) -> PathBuf { Self {
match self { venv_base,
InstallLocation::Venv { venv_base, .. } => { python_version,
if cfg!(windows) {
venv_base.join("Scripts").join("python.exe")
} else {
// canonicalize on python would resolve the symlink
venv_base.join("bin").join("python")
}
}
// TODO: For monotrail use the monotrail launcher
InstallLocation::Monotrail { python, .. } => python.clone(),
} }
} }
pub fn get_python_version(&self) -> (u8, u8) { /// Returns the location of the `python` interpreter.
match self { pub fn python(&self) -> PathBuf {
InstallLocation::Venv { python_version, .. } => *python_version, if cfg!(windows) {
InstallLocation::Monotrail { python_version, .. } => *python_version, self.venv_base.as_ref().join("Scripts").join("python.exe")
} else {
// canonicalize on python would resolve the symlink
self.venv_base.as_ref().join("bin").join("python")
} }
} }
/// TODO: This function is unused? pub fn python_version(&self) -> (u8, u8) {
pub fn is_installed(&self, normalized_name: &str, version: &str) -> bool { self.python_version
match self { }
InstallLocation::Venv {
venv_base, pub fn venv_base(&self) -> &T {
python_version, &self.venv_base
} => {
let site_packages = if cfg!(target_os = "windows") {
venv_base.join("Lib").join("site-packages")
} else {
venv_base
.join("lib")
.join(format!("python{}.{}", python_version.0, python_version.1))
.join("site-packages")
};
site_packages
.join(format!("{normalized_name}-{version}.dist-info"))
.is_dir()
}
InstallLocation::Monotrail { monotrail_root, .. } => monotrail_root
.join(format!("{normalized_name}-{version}"))
.is_dir(),
}
} }
} }
impl InstallLocation<PathBuf> { impl InstallLocation<PathBuf> {
pub fn acquire_lock(&self) -> io::Result<InstallLocation<LockedDir>> { pub fn acquire_lock(&self) -> io::Result<InstallLocation<LockedDir>> {
let root = match self { let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_base)? {
Self::Venv { venv_base, .. } => venv_base,
Self::Monotrail { monotrail_root, .. } => monotrail_root,
};
// If necessary, create monotrail dir
fs::create_dir_all(root)?;
let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(root)? {
locked_dir locked_dir
} else { } else {
warn!( warn!(
"Could not acquire exclusive lock for installing, is another installation process \ "Could not acquire exclusive lock for installing, is another installation process \
running? Sleeping until lock becomes free" running? Sleeping until lock becomes free"
); );
LockedDir::acquire(root)? LockedDir::acquire(&self.venv_base)?
}; };
Ok(match self { Ok(InstallLocation {
Self::Venv { python_version, .. } => InstallLocation::Venv { venv_base: locked_dir,
venv_base: locked_dir, python_version: self.python_version,
python_version: *python_version,
},
Self::Monotrail {
python_version,
python,
..
} => InstallLocation::Monotrail {
monotrail_root: locked_dir,
python: python.clone(),
python_version: *python_version,
},
}) })
} }
} }

2
crates/install-wheel-rs/src/lib.rs
View file

@ -1,4 +1,4 @@
//! Takes a wheel and installs it, either in a venv or for monotrail. //! Takes a wheel and installs it into a venv..
use std::io; use std::io;
use std::io::{Read, Seek}; use std::io::{Read, Seek};

9
crates/install-wheel-rs/src/main.rs
View file

@ -31,10 +31,7 @@ struct Args {
fn main() -> Result<(), Error> { fn main() -> Result<(), Error> {
let args = Args::parse(); let args = Args::parse();
let venv_base = args.venv.canonicalize()?; let venv_base = args.venv.canonicalize()?;
let location = InstallLocation::Venv { let location = InstallLocation::new(venv_base, (args.major, args.minor));
venv_base,
python_version: (args.major, args.minor),
};
let locked_dir = location.acquire_lock()?; let locked_dir = location.acquire_lock()?;
let wheels: Vec<(PathBuf, WheelFilename)> = args let wheels: Vec<(PathBuf, WheelFilename)> = args
@ -69,9 +66,7 @@ fn main() -> Result<(), Error> {
args.compile, args.compile,
!args.skip_hashes, !args.skip_hashes,
&[], &[],
// Only relevant for monotrail style installation location.python(),
"",
location.get_python(),
)?; )?;
Ok(()) Ok(())
}) })

10
crates/install-wheel-rs/src/python_bindings.rs
View file

@ -40,10 +40,10 @@ impl LockedVenv {
#[allow(clippy::needless_pass_by_value)] #[allow(clippy::needless_pass_by_value)]
pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult<Self> { pub(crate) fn new(py: Python, venv: PathBuf) -> PyResult<Self> {
Ok(Self { Ok(Self {
location: InstallLocation::Venv { location: InstallLocation::new(
venv_base: LockedDir::acquire(&venv)?, LockedDir::acquire(&venv)?,
python_version: (py.version_info().major, py.version_info().minor), (py.version_info().major, py.version_info().minor),
}, ),
}) })
} }
@ -65,8 +65,6 @@ impl LockedVenv {
true, true,
true, true,
&[], &[],
// unique_version can be anything since it's only used to monotrail
"",
Path::new(&sys_executable), Path::new(&sys_executable),
) )
})?; })?;

19
crates/install-wheel-rs/src/unpacked.rs
View file

@ -34,26 +34,19 @@ pub fn install_wheel(
let name = &filename.distribution; let name = &filename.distribution;
let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str()); let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str());
let InstallLocation::Venv { let base_location = location.venv_base();
venv_base: base_location,
..
} = location
else {
return Err(Error::InvalidWheel(
"Monotrail installation is not supported yet".to_string(),
));
};
// TODO(charlie): Pass this in. // TODO(charlie): Pass this in.
let site_packages_python = format!( let site_packages_python = format!(
"python{}.{}", "python{}.{}",
location.get_python_version().0, location.python_version().0,
location.get_python_version().1 location.python_version().1
); );
let site_packages = if cfg!(target_os = "windows") { let site_packages = if cfg!(target_os = "windows") {
base_location.join("Lib").join("site-packages") base_location.as_ref().join("Lib").join("site-packages")
} else { } else {
base_location base_location
.as_ref()
.join("lib") .join("lib")
.join(site_packages_python) .join(site_packages_python)
.join("site-packages") .join("site-packages")
@ -94,7 +87,7 @@ pub fn install_wheel(
if data_dir.is_dir() { if data_dir.is_dir() {
debug!(name = name.as_str(), "Installing data"); debug!(name = name.as_str(), "Installing data");
install_data( install_data(
base_location, base_location.as_ref(),
&site_packages, &site_packages,
&data_dir, &data_dir,
&name, &name,

108
crates/install-wheel-rs/src/wheel.rs
View file

@ -11,7 +11,7 @@ use fs_err as fs;
use fs_err::{DirEntry, File}; use fs_err::{DirEntry, File};
use mailparse::MailHeaderMap; use mailparse::MailHeaderMap;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use tempfile::{tempdir, TempDir}; use tempfile::tempdir;
use tracing::{debug, error, span, warn, Level}; use tracing::{debug, error, span, warn, Level};
use walkdir::WalkDir; use walkdir::WalkDir;
use zip::result::ZipError; use zip::result::ZipError;
@ -23,7 +23,7 @@ use wheel_filename::WheelFilename;
use crate::install_location::{InstallLocation, LockedDir}; use crate::install_location::{InstallLocation, LockedDir};
use crate::record::RecordEntry; use crate::record::RecordEntry;
use crate::script::Script; use crate::script::Script;
use crate::{normalize_name, Error}; use crate::Error;
/// `#!/usr/bin/env python` /// `#!/usr/bin/env python`
pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python"; pub const SHEBANG_PYTHON: &str = "#!/usr/bin/env python";
@ -262,25 +262,19 @@ fn unpack_wheel_files<R: Read + Seek>(
} }
pub(crate) fn get_shebang(location: &InstallLocation<LockedDir>) -> String { pub(crate) fn get_shebang(location: &InstallLocation<LockedDir>) -> String {
if matches!(location, InstallLocation::Venv { .. }) { let path = location.python().display().to_string();
let path = location.get_python().display().to_string(); let path = if cfg!(windows) {
let path = if cfg!(windows) { // https://stackoverflow.com/a/50323079
// https://stackoverflow.com/a/50323079 const VERBATIM_PREFIX: &str = r"\\?\";
const VERBATIM_PREFIX: &str = r"\\?\"; if let Some(stripped) = path.strip_prefix(VERBATIM_PREFIX) {
if let Some(stripped) = path.strip_prefix(VERBATIM_PREFIX) { stripped.to_string()
stripped.to_string()
} else {
path
}
} else { } else {
path path
}; }
format!("#!{path}")
} else { } else {
// This will use the monotrail binary moonlighting as python. `python` alone doesn't, path
// we need env to find the python link we put in PATH };
SHEBANG_PYTHON.to_string() format!("#!{path}")
}
} }
/// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual /// To get a launcher on windows we write a minimal .exe launcher binary and then attach the actual
@ -336,8 +330,6 @@ pub(crate) fn write_script_entrypoints(
entrypoints: &[Script], entrypoints: &[Script],
record: &mut Vec<RecordEntry>, record: &mut Vec<RecordEntry>,
) -> Result<(), Error> { ) -> Result<(), Error> {
// for monotrail
fs::create_dir_all(site_packages.join(&bin_rel()))?;
for entrypoint in entrypoints { for entrypoint in entrypoints {
let entrypoint_relative = if cfg!(windows) { let entrypoint_relative = if cfg!(windows) {
// On windows we actually build an .exe wrapper // On windows we actually build an .exe wrapper
@ -677,9 +669,6 @@ fn install_script(
// //
// > The b'#!pythonw' convention is allowed. b'#!pythonw' indicates a GUI script // > The b'#!pythonw' convention is allowed. b'#!pythonw' indicates a GUI script
// > instead of a console script. // > instead of a console script.
//
// We do this in venvs as required, but in monotrail mode we use a fake shebang
// (#!/usr/bin/env python) for injection monotrail as python into PATH later
let placeholder_python = b"#!python"; let placeholder_python = b"#!python";
// scripts might be binaries, so we read an exact number of bytes instead of the first line as string // scripts might be binaries, so we read an exact number of bytes instead of the first line as string
let mut start = Vec::new(); let mut start = Vec::new();
@ -776,11 +765,10 @@ pub(crate) fn install_data(
let target_path = venv_base let target_path = venv_base
.join("include") .join("include")
.join("site") .join("site")
// TODO: Also use just python here in monotrail
.join(format!( .join(format!(
"python{}.{}", "python{}.{}",
location.get_python_version().0, location.python_version().0,
location.get_python_version().1 location.python_version().1
)) ))
.join(dist_name); .join(dist_name);
move_folder_recorded(&data_entry.path(), &target_path, site_packages, record)?; move_folder_recorded(&data_entry.path(), &target_path, site_packages, record)?;
@ -908,51 +896,23 @@ pub fn install_wheel(
// initially used to the console scripts, currently unused. Keeping it because we likely need // initially used to the console scripts, currently unused. Keeping it because we likely need
// it for validation later // it for validation later
_extras: &[String], _extras: &[String],
unique_version: &str,
sys_executable: impl AsRef<Path>, sys_executable: impl AsRef<Path>,
) -> Result<String, Error> { ) -> Result<String, Error> {
let name = &filename.distribution; let name = &filename.distribution;
let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str()); let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str());
let (temp_dir_final_location, base_location) = match location { let base_location = location.venv_base();
InstallLocation::Venv { venv_base, .. } => (None, venv_base.to_path_buf()),
InstallLocation::Monotrail { monotrail_root, .. } => {
let name_version_dir = monotrail_root
.join(normalize_name(name))
.join(unique_version);
fs::create_dir_all(&name_version_dir)?;
let final_location = name_version_dir.join(filename.get_tag());
// temp dir and rename for atomicity
// well, except for windows, because there renaming fails for undeterminable reasons
// with an os error 5 permission denied.
if cfg!(not(windows)) {
let temp_dir = TempDir::new_in(&name_version_dir)?;
let base_location = temp_dir.path().to_path_buf();
(Some((temp_dir, final_location)), base_location)
} else {
fs::create_dir(&final_location)?;
(None, final_location)
}
}
};
let site_packages_python = match location { let site_packages_python = format!(
InstallLocation::Venv { .. } => { "python{}.{}",
format!( location.python_version().0,
"python{}.{}", location.python_version().1
location.get_python_version().0, );
location.get_python_version().1
)
}
// Monotrail installation is for multiple python versions (depending on the wheel tag)
// Potentially needs to be changed to creating pythonx.y symlinks for each python version
// we use it with (on install in that python version)
InstallLocation::Monotrail { .. } => "python".to_string(),
};
let site_packages = if cfg!(target_os = "windows") { let site_packages = if cfg!(target_os = "windows") {
base_location.join("Lib").join("site-packages") base_location.as_ref().join("Lib").join("site-packages")
} else { } else {
base_location base_location
.as_ref()
.join("lib") .join("lib")
.join(site_packages_python) .join(site_packages_python)
.join("site-packages") .join("site-packages")
@ -1014,7 +974,7 @@ pub fn install_wheel(
if data_dir.is_dir() { if data_dir.is_dir() {
debug!(name = name.as_str(), "Installing data"); debug!(name = name.as_str(), "Installing data");
install_data( install_data(
&base_location, base_location.as_ref(),
&site_packages, &site_packages,
&data_dir, &data_dir,
&name, &name,
@ -1022,8 +982,6 @@ pub fn install_wheel(
&console_scripts, &console_scripts,
&gui_scripts, &gui_scripts,
&mut record, &mut record,
// For the monotrail install, we want to keep the fake shebang for our own
// later replacement logic
)?; )?;
// 2.c If applicable, update scripts starting with #!python to point to the correct interpreter. // 2.c If applicable, update scripts starting with #!python to point to the correct interpreter.
// Script are unsupported through data // Script are unsupported through data
@ -1039,7 +997,7 @@ pub fn install_wheel(
bytecode_compile( bytecode_compile(
&site_packages, &site_packages,
unpacked_paths, unpacked_paths,
location.get_python_version(), location.python_version(),
sys_executable.as_ref(), sys_executable.as_ref(),
name.as_str(), name.as_str(),
&mut record, &mut record,
@ -1060,12 +1018,6 @@ pub fn install_wheel(
record_writer.serialize(entry)?; record_writer.serialize(entry)?;
} }
// rename for atomicity
// well, except for windows, see comment above
if let Some((_temp_dir, final_location)) = temp_dir_final_location {
fs::rename(base_location, final_location)?;
}
Ok(filename.get_tag()) Ok(filename.get_tag())
} }
@ -1091,7 +1043,7 @@ fn find_dist_info(
[] => { [] => {
return Err(Error::InvalidWheel( return Err(Error::InvalidWheel(
"Missing .dist-info directory".to_string(), "Missing .dist-info directory".to_string(),
)) ));
} }
[dist_info] => (*dist_info).to_string(), [dist_info] => (*dist_info).to_string(),
_ => { _ => {
@ -1219,7 +1171,7 @@ mod test {
assert_eq!( assert_eq!(
relative_to( relative_to(
Path::new("/home/ferris/carcinization/lib/python/site-packages/foo/__init__.py"), Path::new("/home/ferris/carcinization/lib/python/site-packages/foo/__init__.py"),
Path::new("/home/ferris/carcinization/lib/python/site-packages") Path::new("/home/ferris/carcinization/lib/python/site-packages"),
) )
.unwrap(), .unwrap(),
Path::new("foo/__init__.py") Path::new("foo/__init__.py")
@ -1227,7 +1179,7 @@ mod test {
assert_eq!( assert_eq!(
relative_to( relative_to(
Path::new("/home/ferris/carcinization/lib/marker.txt"), Path::new("/home/ferris/carcinization/lib/marker.txt"),
Path::new("/home/ferris/carcinization/lib/python/site-packages") Path::new("/home/ferris/carcinization/lib/python/site-packages"),
) )
.unwrap(), .unwrap(),
Path::new("../../marker.txt") Path::new("../../marker.txt")
@ -1235,7 +1187,7 @@ mod test {
assert_eq!( assert_eq!(
relative_to( relative_to(
Path::new("/home/ferris/carcinization/bin/foo_launcher"), Path::new("/home/ferris/carcinization/bin/foo_launcher"),
Path::new("/home/ferris/carcinization/lib/python/site-packages") Path::new("/home/ferris/carcinization/lib/python/site-packages"),
) )
.unwrap(), .unwrap(),
Path::new("../../../bin/foo_launcher") Path::new("../../../bin/foo_launcher")
@ -1256,7 +1208,7 @@ mod test {
Script::from_value( Script::from_value(
"launcher", "launcher",
"foo.bar:main", "foo.bar:main",
Some(&["bar".to_string(), "baz".to_string()]) Some(&["bar".to_string(), "baz".to_string()]),
) )
.unwrap(), .unwrap(),
Some(Script { Some(Script {
@ -1273,7 +1225,7 @@ mod test {
Script::from_value( Script::from_value(
"launcher", "launcher",
"foomod:main_bar [bar,baz]", "foomod:main_bar [bar,baz]",
Some(&["bar".to_string(), "baz".to_string()]) Some(&["bar".to_string(), "baz".to_string()]),
) )
.unwrap(), .unwrap(),
Some(Script { Some(Script {

5
crates/puffin-installer/src/lib.rs
View file

@ -114,10 +114,7 @@ pub async fn install(
); );
// Phase 3: Install each wheel. // Phase 3: Install each wheel.
let location = InstallLocation::Venv { let location = InstallLocation::new(python.venv().to_path_buf(), python.simple_version());
venv_base: python.venv().to_path_buf(),
python_version: python.simple_version(),
};
let locked_dir = location.acquire_lock()?; let locked_dir = location.acquire_lock()?;
for wheel in wheels { for wheel in wheels {