mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-09-28 21:14:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Remove refresh checks from the install plan (#1119)

Browse source 
## Summary

Rather than checking cache freshness in the install plan, it's a lot
simple to have the install plan _never_ return cached data when the
refresh policy is in place, and then rely on the distribution database
to check for freshness. The original implementation didn't support this,
since the distribution database was rebuilding things too often. Now, it
rarely rebuilds (it's much better about this), so it seems conceptually
much simpler to split up the responsibilities like this.
This commit is contained in:
Charlie Marsh 2024-01-25 19:48:16 -08:00 • committed by GitHub
parent 50057cd5f2
commit f593b65447
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 57 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/puffin-cache/src/lib.rs
View file

@ -160,6 +160,15 @@ impl Cache {
CacheEntry::new(self.bucket(cache_bucket).join(dir), file)
}
/// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
pub fn must_revalidate(&self, package: &PackageName) -> bool {
match &self.refresh {
Refresh::None => false,
Refresh::All(_) => true,
Refresh::Packages(packages, _) => packages.contains(package),
}
}
/// Returns `true` if a cache entry is up-to-date given the [`Refresh`] policy.
pub fn freshness(
&self,

35
crates/puffin-distribution/src/index/built_wheel_index.rs
View file

@ -1,8 +1,7 @@
use distribution_types::{git_reference, DirectUrlSourceDist, GitSourceDist, Name, PathSourceDist};
use platform_tags::Tags;
use puffin_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, Freshness, WheelCache};
use puffin_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, WheelCache};
use puffin_fs::symlinks;
use puffin_normalize::PackageName;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, read_timestamp_manifest, MANIFEST};
@ -34,12 +33,7 @@ impl BuiltWheelIndex {
return Ok(None);
};
Ok(Self::find(
&cache_shard.shard(manifest.id()),
source_dist.name(),
cache,
tags,
))
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a local path.
@ -66,12 +60,7 @@ impl BuiltWheelIndex {
return Ok(None);
};
Ok(Self::find(
&cache_shard.shard(manifest.id()),
source_dist.name(),
cache,
tags,
))
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL.
@ -86,7 +75,7 @@ impl BuiltWheelIndex {
.remote_wheel_dir(source_dist.name().as_ref()),
);
Self::find(&cache_shard, source_dist.name(), cache, tags)
Self::find(&cache_shard, tags)
}
/// Find the "best" distribution in the index for a given source distribution.
@ -105,12 +94,7 @@ impl BuiltWheelIndex {
/// ```
///
/// The `shard` should be `built-wheels-v0/pypi/django-allauth-0.51.0.tar.gz`.
fn find(
shard: &CacheShard,
package: &PackageName,
cache: &Cache,
tags: &Tags,
) -> Option<CachedWheel> {
fn find(shard: &CacheShard, tags: &Tags) -> Option<CachedWheel> {
let mut candidate: Option<CachedWheel> = None;
// Unzipped wheels are stored as symlinks into the archive directory.
@ -118,15 +102,6 @@ impl BuiltWheelIndex {
match CachedWheel::from_path(&subdir) {
None => {}
Some(dist_info) => {
// If the [`Refresh`] policy is set, ignore entries that were created before
// the cutoff.
if cache
.freshness(&dist_info.entry, Some(package))
.is_ok_and(Freshness::is_stale)
{
continue;
}
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);

21
crates/puffin-distribution/src/index/registry_wheel_index.rs
View file

@ -7,7 +7,7 @@ use rustc_hash::FxHashMap;
use distribution_types::{CachedRegistryDist, FlatIndexLocation, IndexLocations, IndexUrl};
use pep440_rs::Version;
use platform_tags::Tags;
use puffin_cache::{Cache, CacheBucket, Freshness, WheelCache};
use puffin_cache::{Cache, CacheBucket, WheelCache};
use puffin_fs::{directories, symlinks};
use puffin_normalize::PackageName;
@ -94,7 +94,7 @@ impl<'a> RegistryWheelIndex<'a> {
WheelCache::Index(index_url).remote_wheel_dir(package.to_string()),
);
Self::add_directory(&wheel_dir, package, cache, tags, &mut versions);
Self::add_directory(&wheel_dir, tags, &mut versions);
// Index all the built wheels, created by downloading and building source distributions
// from the registry.
@ -109,13 +109,7 @@ impl<'a> RegistryWheelIndex<'a> {
let cache_shard = cache_shard.shard(shard);
let manifest_entry = cache_shard.entry(MANIFEST);
if let Ok(Some(manifest)) = read_http_manifest(&manifest_entry) {
Self::add_directory(
cache_shard.join(manifest.id()),
package,
cache,
tags,
&mut versions,
);
Self::add_directory(cache_shard.join(manifest.id()), tags, &mut versions);
};
}
}
@ -128,8 +122,6 @@ impl<'a> RegistryWheelIndex<'a> {
/// Each subdirectory in the given path is expected to be that of an unzipped wheel.
fn add_directory(
path: impl AsRef<Path>,
package: &PackageName,
cache: &Cache,
tags: &Tags,
versions: &mut BTreeMap<Version, CachedRegistryDist>,
) {
@ -138,13 +130,6 @@ impl<'a> RegistryWheelIndex<'a> {
match CachedWheel::from_path(&wheel_dir) {
None => {}
Some(dist_info) => {
if cache
.freshness(&dist_info.entry, Some(package))
.is_ok_and(Freshness::is_stale)
{
continue;
}
let dist_info = dist_info.into_registry_dist();
// Pick the wheel with the highest priority

31
crates/puffin-installer/src/plan.rs
View file

@ -12,10 +12,9 @@ use distribution_types::{
};
use pep508_rs::{Requirement, VersionOrUrl};
use platform_tags::Tags;
use puffin_cache::{
ArchiveTimestamp, Cache, CacheBucket, CacheEntry, Freshness, Timestamp, WheelCache,
};
use puffin_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheEntry, Timestamp, WheelCache};
use puffin_distribution::{BuiltWheelIndex, RegistryWheelIndex};
use puffin_fs::NormalizedDisplay;
use puffin_interpreter::Virtualenv;
use puffin_normalize::PackageName;
use puffin_traits::NoBinary;
@ -186,6 +185,12 @@ impl<'a> Planner<'a> {
}
}
if cache.must_revalidate(&requirement.name) {
debug!("Must revalidate requirement: {requirement}");
remote.push(requirement.clone());
continue;
}
// Identify any locally-available distributions that satisfy the requirement.
match requirement.version_or_url.as_ref() {
None => {
@ -247,10 +252,6 @@ impl<'a> Planner<'a> {
)
.entry(wheel.filename.stem());
if cache
.freshness(&cache_entry, Some(wheel.name()))
.is_ok_and(Freshness::is_fresh)
{
match cache_entry.path().canonicalize() {
Ok(archive) => {
let cached_dist = CachedDirectUrlDist::from_url(
@ -259,9 +260,7 @@ impl<'a> Planner<'a> {
archive,
);
debug!(
"URL wheel requirement already cached: {cached_dist}"
);
debug!("URL wheel requirement already cached: {cached_dist}");
local.push(CachedDist::Url(cached_dist));
continue;
}
@ -271,12 +270,11 @@ impl<'a> Planner<'a> {
Err(err) => return Err(err.into()),
}
}
}
Dist::Built(BuiltDist::Path(wheel)) => {
if !wheel.filename.is_compatible(tags) {
bail!(
"A path dependency is incompatible with the current platform: {}",
wheel.path.display()
wheel.path.normalized_display()
);
}
@ -297,10 +295,6 @@ impl<'a> Planner<'a> {
)
.entry(wheel.filename.stem());
if cache
.freshness(&cache_entry, Some(wheel.name()))
.is_ok_and(Freshness::is_fresh)
{
if not_modified_cache(&cache_entry, &wheel.path)? {
match cache_entry.path().canonicalize() {
Ok(archive) => {
@ -310,7 +304,9 @@ impl<'a> Planner<'a> {
archive,
);
debug!("URL wheel requirement already cached: {cached_dist}");
debug!(
"URL wheel requirement already cached: {cached_dist}"
);
local.push(CachedDist::Url(cached_dist));
continue;
}
@ -321,7 +317,6 @@ impl<'a> Planner<'a> {
}
}
}
}
Dist::Source(SourceDist::DirectUrl(sdist)) => {
// Find the most-compatible wheel from the cache, since we don't know
// the filename in advance.