mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-10-31 12:06:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add --find-links source distributions to the registry cache (#2986)

Browse source 
## Summary

Source distributions in `--find-links` are now properly picked up in the
cache.

Closes https://github.com/astral-sh/uv/issues/2978.
This commit is contained in:
Charlie Marsh 2024-04-10 21:25:58 -04:00 • committed by GitHub
parent 32f129c245
commit 3dd673677a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 182 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/distribution-types/src/index_url.rs
View file

@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
pub enum IndexUrl {
Pypi(VerbatimUrl),
Url(VerbatimUrl),
Path(VerbatimUrl),
}
impl IndexUrl {
@ -32,6 +33,7 @@ impl IndexUrl {
match self {
Self::Pypi(url) => url.raw(),
Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
}
}
}
@ -41,6 +43,7 @@ impl Display for IndexUrl {
match self {
Self::Pypi(url) => Display::fmt(url, f),
Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
}
}
}
@ -50,6 +53,7 @@ impl Verbatim for IndexUrl {
match self {
Self::Pypi(url) => url.verbatim(),
Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
}
}
}
@ -83,6 +87,7 @@ impl From<IndexUrl> for Url {
match index {
IndexUrl::Pypi(url) => url.to_url(),
IndexUrl::Url(url) => url.to_url(),
IndexUrl::Path(url) => url.to_url(),
}
}
}
@ -94,6 +99,7 @@ impl Deref for IndexUrl {
match &self {
Self::Pypi(url) => url,
Self::Url(url) => url,
Self::Path(url) => url,
}
}
}

2
crates/uv-client/src/flat_index.rs
View file

@ -205,7 +205,7 @@ impl<'a> FlatIndexClient<'a> {
fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
// Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?;
let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path));
let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path));
let mut dists = Vec::new();
for entry in fs_err::read_dir(path)? {

1
crates/uv-client/src/registry_client.rs
View file

@ -285,6 +285,7 @@ impl RegistryClient {
Path::new(&match index {
IndexUrl::Pypi(_) => "pypi".to_string(),
IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
}),
format!("{package_name}.rkyv"),
);

15
crates/uv-distribution/src/index/cached_wheel.rs
View file

@ -20,9 +20,14 @@ pub struct CachedWheel {
impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_built_source(path: &Path) -> Option<Self> {
pub fn from_built_source(path: impl AsRef<Path>) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Convert to a cached wheel.
let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive);
let hashes = Vec::new();
@ -54,7 +59,9 @@ impl CachedWheel {
}
/// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`).
pub fn from_http_pointer(path: &Path, cache: &Cache) -> Option<Self> {
pub fn from_http_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
@ -73,7 +80,9 @@ impl CachedWheel {
}
/// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`).
pub fn from_local_pointer(path: &Path, cache: &Cache) -> Option<Self> {
pub fn from_local_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;

48
crates/uv-distribution/src/index/registry_wheel_index.rs
View file

@ -13,7 +13,7 @@ use uv_normalize::PackageName;
use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{HttpRevisionPointer, HTTP_REVISION};
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
/// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)]
@ -88,13 +88,13 @@ impl<'a> RegistryWheelIndex<'a> {
) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new();
// Collect into owned `IndexUrl`
// Collect into owned `IndexUrl`.
let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index()
.filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(path) => {
let path = fs_err::canonicalize(path).ok()?;
Some(IndexUrl::Url(VerbatimUrl::from_path(path)))
Some(IndexUrl::Path(VerbatimUrl::from_path(path)))
}
FlatIndexLocation::Url(url) => {
Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
@ -112,12 +112,15 @@ impl<'a> RegistryWheelIndex<'a> {
// For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http`
// or `<index>/<package-name>/<version>/<wheel>.rev`.
for file in files(&wheel_dir) {
match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{
if let Some(wheel) =
CachedWheel::from_http_pointer(&wheel_dir.join(&file), cache)
CachedWheel::from_http_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
@ -125,13 +128,15 @@ impl<'a> RegistryWheelIndex<'a> {
}
}
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{
if let Some(wheel) =
CachedWheel::from_local_pointer(&wheel_dir.join(&file), cache)
CachedWheel::from_local_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
@ -140,6 +145,8 @@ impl<'a> RegistryWheelIndex<'a> {
}
}
}
}
}
// Index all the built wheels, created by downloading and building source distributions
// from the registry.
@ -152,18 +159,39 @@ impl<'a> RegistryWheelIndex<'a> {
for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard);
// Read the revision from the cache.
let revision = match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(&revision_entry) {
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
let revision_entry = cache_shard.entry(LOCAL_REVISION);
if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
};
if let Some(revision) = revision {
// Enforce hash-checking based on the source distribution.
let revision = pointer.into_revision();
if revision.satisfies(hasher.get(package)) {
for wheel_dir in symlinks(cache_shard.join(revision.id())) {
if let Some(wheel) = CachedWheel::from_built_source(&wheel_dir) {
if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
};
}
}
}

84
crates/uv-distribution/src/source/mod.rs
View file

@ -87,6 +87,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
) -> Result<BuiltWheelMetadata, Error> {
let built_wheel_metadata = match &source {
BuildableSource::Dist(SourceDist::Registry(dist)) => {
// For registry source distributions, shard by package, then version, for
// convenience in debugging.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
let url = match &dist.file.url {
FileLocation::RelativeUrl(base, url) => {
pypi_types::base_url_join_relative(base, url)?
@ -103,6 +112,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
url: &url,
path: Cow::Borrowed(path),
},
&cache_shard,
tags,
hashes,
)
@ -111,15 +121,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
};
// For registry source distributions, shard by package, then version, for
// convenience in debugging.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
self.url(
source,
&dist.file.filename,
@ -165,7 +166,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed()
.await?
} else {
self.archive(source, &PathSourceUrl::from(dist), tags, hashes)
let cache_shard = self
.build_context
.cache()
.shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root());
self.archive(
source,
&PathSourceUrl::from(dist),
&cache_shard,
tags,
hashes,
)
.boxed()
.await?
}
@ -204,7 +215,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed()
.await?
} else {
self.archive(source, resource, tags, hashes).boxed().await?
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
self.archive(source, resource, &cache_shard, tags, hashes)
.boxed()
.await?
}
}
};
@ -222,6 +239,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
) -> Result<ArchiveMetadata, Error> {
let metadata = match &source {
BuildableSource::Dist(SourceDist::Registry(dist)) => {
// For registry source distributions, shard by package, then version.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
let url = match &dist.file.url {
FileLocation::RelativeUrl(base, url) => {
pypi_types::base_url_join_relative(base, url)?
@ -238,6 +263,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
url: &url,
path: Cow::Borrowed(path),
},
&cache_shard,
hashes,
)
.boxed()
@ -245,14 +271,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
};
// For registry source distributions, shard by package, then version.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
self.url_metadata(
source,
&dist.file.filename,
@ -296,7 +314,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed()
.await?
} else {
self.archive_metadata(source, &PathSourceUrl::from(dist), hashes)
let cache_shard = self
.build_context
.cache()
.shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root());
self.archive_metadata(source, &PathSourceUrl::from(dist), &cache_shard, hashes)
.boxed()
.await?
}
@ -334,7 +356,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed()
.await?
} else {
self.archive_metadata(source, resource, hashes)
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
self.archive_metadata(source, resource, &cache_shard, hashes)
.boxed()
.await?
}
@ -573,17 +599,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self,
source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>,
cache_shard: &CacheShard,
tags: &Tags,
hashes: HashPolicy<'_>,
) -> Result<BuiltWheelMetadata, Error> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
// Fetch the revision for the source distribution.
let revision = self
.archive_revision(source, resource, &cache_shard, hashes)
.archive_revision(source, resource, cache_shard, hashes)
.await?;
// Before running the build, check that the hashes match.
@ -644,16 +666,12 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self,
source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>,
cache_shard: &CacheShard,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
// Fetch the revision for the source distribution.
let revision = self
.archive_revision(source, resource, &cache_shard, hashes)
.archive_revision(source, resource, cache_shard, hashes)
.await?;
// Before running the build, check that the hashes match.

53
crates/uv/tests/pip_sync.rs
View file

@ -2539,12 +2539,12 @@ fn find_links_offline_no_match() -> Result<()> {
/// Sync using `--find-links` with a local directory. Ensure that cached wheels are reused.
#[test]
fn find_links_cache() -> Result<()> {
fn find_links_wheel_cache() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_txt = context.temp_dir.child("requirements.txt");
requirements_txt.write_str(indoc! {r"
tqdm
tqdm==1000.0.0
"})?;
// Install `tqdm`.
@ -2585,6 +2585,55 @@ fn find_links_cache() -> Result<()> {
Ok(())
}
/// Sync using `--find-links` with a local directory. Ensure that cached source distributions are
/// reused.
#[test]
fn find_links_source_cache() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_txt = context.temp_dir.child("requirements.txt");
requirements_txt.write_str(indoc! {r"
tqdm==999.0.0
"})?;
// Install `tqdm`.
uv_snapshot!(context.filters(), command(&context)
.arg("requirements.txt")
.arg("--find-links")
.arg(context.workspace_root.join("scripts/links/")), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
+ tqdm==999.0.0
"###
);
// Reinstall `tqdm` with `--reinstall`. Ensure that the wheel is reused.
uv_snapshot!(context.filters(), command(&context)
.arg("requirements.txt")
.arg("--reinstall")
.arg("--find-links")
.arg(context.workspace_root.join("scripts/links/")), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Uninstalled 1 package in [TIME]
Installed 1 package in [TIME]
- tqdm==999.0.0
+ tqdm==999.0.0
"###
);
Ok(())
}
/// Install without network access via the `--offline` flag.
#[test]
fn offline() -> Result<()> {