Store cached wheels by dist-info-like name (#52)

Closes https://github.com/astral-sh/puffin/issues/50.
This commit is contained in:
Charlie Marsh 2023-10-08 00:28:04 -04:00 committed by GitHub
parent fd5aef2c75
commit 5eef6e9636
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 14 deletions

1
Cargo.lock generated
View file

@ -1720,6 +1720,7 @@ dependencies = [
"install-wheel-rs", "install-wheel-rs",
"puffin-client", "puffin-client",
"puffin-interpreter", "puffin-interpreter",
"puffin-package",
"rayon", "rayon",
"tempfile", "tempfile",
"tokio", "tokio",

View file

@ -13,6 +13,7 @@ license.workspace = true
install-wheel-rs = { path = "../install-wheel-rs", default-features = false } install-wheel-rs = { path = "../install-wheel-rs", default-features = false }
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" } puffin-interpreter = { path = "../puffin-interpreter" }
puffin-package = { path = "../puffin-package" }
wheel-filename = { path = "../wheel-filename" } wheel-filename = { path = "../wheel-filename" }
anyhow = { workspace = true } anyhow = { workspace = true }

View file

@ -14,12 +14,15 @@ use zip::ZipArchive;
use install_wheel_rs::{unpacked, InstallLocation}; use install_wheel_rs::{unpacked, InstallLocation};
use puffin_client::{File, PypiClient}; use puffin_client::{File, PypiClient};
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use puffin_package::package_name::PackageName;
use wheel_filename::WheelFilename; use wheel_filename::WheelFilename;
use crate::vendor::CloneableSeekableReader; use crate::vendor::CloneableSeekableReader;
mod vendor; mod vendor;
static WHEEL_CACHE: &str = "wheels-v0";
/// Install a set of wheels into a Python virtual environment. /// Install a set of wheels into a Python virtual environment.
pub async fn install( pub async fn install(
wheels: &[File], wheels: &[File],
@ -27,23 +30,26 @@ pub async fn install(
client: &PypiClient, client: &PypiClient,
cache: Option<&Path>, cache: Option<&Path>,
) -> Result<()> { ) -> Result<()> {
// Create the cache subdirectory, if necessary.
if let Some(cache) = cache {
tokio::fs::create_dir_all(cache.join(WHEEL_CACHE)).await?;
}
// Phase 1: Fetch the wheels in parallel. // Phase 1: Fetch the wheels in parallel.
debug!("Phase 1: Fetching wheels"); debug!("Phase 1: Fetching wheels");
let mut fetches = JoinSet::new(); let mut fetches = JoinSet::new();
let mut downloads = Vec::with_capacity(wheels.len()); let mut downloads = Vec::with_capacity(wheels.len());
for wheel in wheels { for wheel in wheels {
let sha256 = wheel.hashes.sha256.clone();
let filename = wheel.filename.clone();
// If the unzipped wheel exists in the cache, skip it. // If the unzipped wheel exists in the cache, skip it.
let key = cache_key(wheel)?;
if let Some(cache) = cache { if let Some(cache) = cache {
if cache.join(&sha256).exists() { if cache.join(WHEEL_CACHE).join(&key).exists() {
debug!("Found wheel in cache: {:?}", filename); debug!("Found wheel in cache: {:?}", wheel.filename);
continue; continue;
} }
} }
debug!("Fetching wheel: {:?}", filename); debug!("Fetching wheel: {:?}", wheel.filename);
fetches.spawn(fetch_wheel( fetches.spawn(fetch_wheel(
wheel.clone(), wheel.clone(),
@ -60,14 +66,14 @@ pub async fn install(
// Phase 2: Unpack the wheels into the cache. // Phase 2: Unpack the wheels into the cache.
debug!("Phase 2: Unpacking wheels"); debug!("Phase 2: Unpacking wheels");
for wheel in downloads { for wheel in downloads {
let sha256 = wheel.file.hashes.sha256.clone();
let filename = wheel.file.filename.clone(); let filename = wheel.file.filename.clone();
let key = cache_key(&wheel.file)?;
debug!("Unpacking wheel: {:?}", filename); debug!("Unpacking wheel: {:?}", filename);
// Unzip the wheel. // Unzip the wheel.
tokio::task::spawn_blocking({ tokio::task::spawn_blocking({
let target = temp_dir.path().join(&sha256); let target = temp_dir.path().join(&key);
move || unzip_wheel(wheel, &target) move || unzip_wheel(wheel, &target)
}) })
.await??; .await??;
@ -75,7 +81,11 @@ pub async fn install(
// Write the unzipped wheel to the cache (atomically). // Write the unzipped wheel to the cache (atomically).
if let Some(cache) = cache { if let Some(cache) = cache {
debug!("Caching wheel: {:?}", filename); debug!("Caching wheel: {:?}", filename);
tokio::fs::rename(temp_dir.path().join(&sha256), cache.join(&sha256)).await?; tokio::fs::rename(
temp_dir.path().join(&key),
cache.join(WHEEL_CACHE).join(&key),
)
.await?;
} }
} }
@ -88,18 +98,33 @@ pub async fn install(
let locked_dir = location.acquire_lock()?; let locked_dir = location.acquire_lock()?;
for wheel in wheels { for wheel in wheels {
let dir = cache let key = cache_key(wheel)?;
.unwrap_or_else(|| temp_dir.path()) let dir = cache.map_or_else(
.join(&wheel.hashes.sha256); || temp_dir.path().join(&key),
let filename = WheelFilename::from_str(&wheel.filename)?; |cache| cache.join(WHEEL_CACHE).join(&key),
);
let wheel_filename = WheelFilename::from_str(&wheel.filename)?;
// TODO(charlie): Should this be async? // TODO(charlie): Should this be async?
unpacked::install_wheel(&locked_dir, &dir, &filename)?; unpacked::install_wheel(&locked_dir, &dir, &wheel_filename)?;
} }
Ok(()) Ok(())
} }
/// Return the cache key for an unzipped wheel. The cache key should be equivalent to the
/// `.dist-info` directory name, i.e., `<name>-<version>.dist-info`, where `name` is the
/// normalized package name.
fn cache_key(wheel: &File) -> Result<String> {
let filename = WheelFilename::from_str(&wheel.filename)?;
Ok(format!(
"{}-{}",
PackageName::normalize(filename.distribution),
filename.version
))
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct FetchedWheel { struct FetchedWheel {
file: File, file: File,