Add --find-links source distributions to the registry cache (#2986)

## Summary

Source distributions in `--find-links` are now properly picked up in the
cache.

Closes https://github.com/astral-sh/uv/issues/2978.
This commit is contained in:
Charlie Marsh 2024-04-10 21:25:58 -04:00 committed by GitHub
parent 32f129c245
commit 3dd673677a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 182 additions and 71 deletions

View file

@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
pub enum IndexUrl { pub enum IndexUrl {
Pypi(VerbatimUrl), Pypi(VerbatimUrl),
Url(VerbatimUrl), Url(VerbatimUrl),
Path(VerbatimUrl),
} }
impl IndexUrl { impl IndexUrl {
@ -32,6 +33,7 @@ impl IndexUrl {
match self { match self {
Self::Pypi(url) => url.raw(), Self::Pypi(url) => url.raw(),
Self::Url(url) => url.raw(), Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
} }
} }
} }
@ -41,6 +43,7 @@ impl Display for IndexUrl {
match self { match self {
Self::Pypi(url) => Display::fmt(url, f), Self::Pypi(url) => Display::fmt(url, f),
Self::Url(url) => Display::fmt(url, f), Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
} }
} }
} }
@ -50,6 +53,7 @@ impl Verbatim for IndexUrl {
match self { match self {
Self::Pypi(url) => url.verbatim(), Self::Pypi(url) => url.verbatim(),
Self::Url(url) => url.verbatim(), Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
} }
} }
} }
@ -83,6 +87,7 @@ impl From<IndexUrl> for Url {
match index { match index {
IndexUrl::Pypi(url) => url.to_url(), IndexUrl::Pypi(url) => url.to_url(),
IndexUrl::Url(url) => url.to_url(), IndexUrl::Url(url) => url.to_url(),
IndexUrl::Path(url) => url.to_url(),
} }
} }
} }
@ -94,6 +99,7 @@ impl Deref for IndexUrl {
match &self { match &self {
Self::Pypi(url) => url, Self::Pypi(url) => url,
Self::Url(url) => url, Self::Url(url) => url,
Self::Path(url) => url,
} }
} }
} }

View file

@ -205,7 +205,7 @@ impl<'a> FlatIndexClient<'a> {
fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> { fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
// Absolute paths are required for the URL conversion. // Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?; let path = fs_err::canonicalize(path)?;
let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path)); let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path));
let mut dists = Vec::new(); let mut dists = Vec::new();
for entry in fs_err::read_dir(path)? { for entry in fs_err::read_dir(path)? {

View file

@ -285,6 +285,7 @@ impl RegistryClient {
Path::new(&match index { Path::new(&match index {
IndexUrl::Pypi(_) => "pypi".to_string(), IndexUrl::Pypi(_) => "pypi".to_string(),
IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
}), }),
format!("{package_name}.rkyv"), format!("{package_name}.rkyv"),
); );

View file

@ -20,9 +20,14 @@ pub struct CachedWheel {
impl CachedWheel { impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`). /// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_built_source(path: &Path) -> Option<Self> { pub fn from_built_source(path: impl AsRef<Path>) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?; let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?; let filename = WheelFilename::from_stem(filename).ok()?;
// Convert to a cached wheel.
let archive = path.canonicalize().ok()?; let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive); let entry = CacheEntry::from_path(archive);
let hashes = Vec::new(); let hashes = Vec::new();
@ -54,7 +59,9 @@ impl CachedWheel {
} }
/// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`). /// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`).
pub fn from_http_pointer(path: &Path, cache: &Cache) -> Option<Self> { pub fn from_http_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename. // Determine the wheel filename.
let filename = path.file_name()?.to_str()?; let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?; let filename = WheelFilename::from_stem(filename).ok()?;
@ -73,7 +80,9 @@ impl CachedWheel {
} }
/// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`). /// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`).
pub fn from_local_pointer(path: &Path, cache: &Cache) -> Option<Self> { pub fn from_local_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename. // Determine the wheel filename.
let filename = path.file_name()?.to_str()?; let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?; let filename = WheelFilename::from_stem(filename).ok()?;

View file

@ -13,7 +13,7 @@ use uv_normalize::PackageName;
use uv_types::HashStrategy; use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel; use crate::index::cached_wheel::CachedWheel;
use crate::source::{HttpRevisionPointer, HTTP_REVISION}; use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
/// A local index of distributions that originate from a registry, like `PyPI`. /// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)] #[derive(Debug)]
@ -88,13 +88,13 @@ impl<'a> RegistryWheelIndex<'a> {
) -> BTreeMap<Version, CachedRegistryDist> { ) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new(); let mut versions = BTreeMap::new();
// Collect into owned `IndexUrl` // Collect into owned `IndexUrl`.
let flat_index_urls: Vec<IndexUrl> = index_locations let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index() .flat_index()
.filter_map(|flat_index| match flat_index { .filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(path) => { FlatIndexLocation::Path(path) => {
let path = fs_err::canonicalize(path).ok()?; let path = fs_err::canonicalize(path).ok()?;
Some(IndexUrl::Url(VerbatimUrl::from_path(path))) Some(IndexUrl::Path(VerbatimUrl::from_path(path)))
} }
FlatIndexLocation::Url(url) => { FlatIndexLocation::Url(url) => {
Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone()))) Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
@ -112,30 +112,37 @@ impl<'a> RegistryWheelIndex<'a> {
// For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http` // For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http`
// or `<index>/<package-name>/<version>/<wheel>.rev`. // or `<index>/<package-name>/<version>/<wheel>.rev`.
for file in files(&wheel_dir) { for file in files(&wheel_dir) {
if file match index_url {
.extension() // Add files from remote registries.
.is_some_and(|ext| ext.eq_ignore_ascii_case("http")) IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
{ if file
if let Some(wheel) = .extension()
CachedWheel::from_http_pointer(&wheel_dir.join(&file), cache) .is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{ {
// Enforce hash-checking based on the built distribution. if let Some(wheel) =
if wheel.satisfies(hasher.get(package)) { CachedWheel::from_http_pointer(wheel_dir.join(file), cache)
Self::add_wheel(wheel, tags, &mut versions); {
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
} }
} }
} // Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
if file if file
.extension() .extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev")) .is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{ {
if let Some(wheel) = if let Some(wheel) =
CachedWheel::from_local_pointer(&wheel_dir.join(&file), cache) CachedWheel::from_local_pointer(wheel_dir.join(file), cache)
{ {
// Enforce hash-checking based on the built distribution. // Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) { if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions); Self::add_wheel(wheel, tags, &mut versions);
}
}
} }
} }
} }
@ -152,18 +159,39 @@ impl<'a> RegistryWheelIndex<'a> {
for shard in directories(&cache_shard) { for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists. // Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard); let cache_shard = cache_shard.shard(shard);
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(&revision_entry) { // Read the revision from the cache.
let revision = match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
let revision_entry = cache_shard.entry(LOCAL_REVISION);
if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
};
if let Some(revision) = revision {
// Enforce hash-checking based on the source distribution. // Enforce hash-checking based on the source distribution.
let revision = pointer.into_revision();
if revision.satisfies(hasher.get(package)) { if revision.satisfies(hasher.get(package)) {
for wheel_dir in symlinks(cache_shard.join(revision.id())) { for wheel_dir in symlinks(cache_shard.join(revision.id())) {
if let Some(wheel) = CachedWheel::from_built_source(&wheel_dir) { if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
Self::add_wheel(wheel, tags, &mut versions); Self::add_wheel(wheel, tags, &mut versions);
} }
} }
} }
}; }
} }
} }

View file

@ -87,6 +87,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
) -> Result<BuiltWheelMetadata, Error> { ) -> Result<BuiltWheelMetadata, Error> {
let built_wheel_metadata = match &source { let built_wheel_metadata = match &source {
BuildableSource::Dist(SourceDist::Registry(dist)) => { BuildableSource::Dist(SourceDist::Registry(dist)) => {
// For registry source distributions, shard by package, then version, for
// convenience in debugging.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
let url = match &dist.file.url { let url = match &dist.file.url {
FileLocation::RelativeUrl(base, url) => { FileLocation::RelativeUrl(base, url) => {
pypi_types::base_url_join_relative(base, url)? pypi_types::base_url_join_relative(base, url)?
@ -103,6 +112,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
url: &url, url: &url,
path: Cow::Borrowed(path), path: Cow::Borrowed(path),
}, },
&cache_shard,
tags, tags,
hashes, hashes,
) )
@ -111,15 +121,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} }
}; };
// For registry source distributions, shard by package, then version, for
// convenience in debugging.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
self.url( self.url(
source, source,
&dist.file.filename, &dist.file.filename,
@ -165,9 +166,19 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
} else { } else {
self.archive(source, &PathSourceUrl::from(dist), tags, hashes) let cache_shard = self
.boxed() .build_context
.await? .cache()
.shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root());
self.archive(
source,
&PathSourceUrl::from(dist),
&cache_shard,
tags,
hashes,
)
.boxed()
.await?
} }
} }
BuildableSource::Url(SourceUrl::Direct(resource)) => { BuildableSource::Url(SourceUrl::Direct(resource)) => {
@ -204,7 +215,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
} else { } else {
self.archive(source, resource, tags, hashes).boxed().await? let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
self.archive(source, resource, &cache_shard, tags, hashes)
.boxed()
.await?
} }
} }
}; };
@ -222,6 +239,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
) -> Result<ArchiveMetadata, Error> { ) -> Result<ArchiveMetadata, Error> {
let metadata = match &source { let metadata = match &source {
BuildableSource::Dist(SourceDist::Registry(dist)) => { BuildableSource::Dist(SourceDist::Registry(dist)) => {
// For registry source distributions, shard by package, then version.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
let url = match &dist.file.url { let url = match &dist.file.url {
FileLocation::RelativeUrl(base, url) => { FileLocation::RelativeUrl(base, url) => {
pypi_types::base_url_join_relative(base, url)? pypi_types::base_url_join_relative(base, url)?
@ -238,6 +263,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
url: &url, url: &url,
path: Cow::Borrowed(path), path: Cow::Borrowed(path),
}, },
&cache_shard,
hashes, hashes,
) )
.boxed() .boxed()
@ -245,14 +271,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} }
}; };
// For registry source distributions, shard by package, then version.
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Index(&dist.index)
.wheel_dir(dist.filename.name.as_ref())
.join(dist.filename.version.to_string()),
);
self.url_metadata( self.url_metadata(
source, source,
&dist.file.filename, &dist.file.filename,
@ -296,7 +314,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
} else { } else {
self.archive_metadata(source, &PathSourceUrl::from(dist), hashes) let cache_shard = self
.build_context
.cache()
.shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root());
self.archive_metadata(source, &PathSourceUrl::from(dist), &cache_shard, hashes)
.boxed() .boxed()
.await? .await?
} }
@ -334,7 +356,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
} else { } else {
self.archive_metadata(source, resource, hashes) let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
self.archive_metadata(source, resource, &cache_shard, hashes)
.boxed() .boxed()
.await? .await?
} }
@ -573,17 +599,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>, resource: &PathSourceUrl<'_>,
cache_shard: &CacheShard,
tags: &Tags, tags: &Tags,
hashes: HashPolicy<'_>, hashes: HashPolicy<'_>,
) -> Result<BuiltWheelMetadata, Error> { ) -> Result<BuiltWheelMetadata, Error> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
// Fetch the revision for the source distribution. // Fetch the revision for the source distribution.
let revision = self let revision = self
.archive_revision(source, resource, &cache_shard, hashes) .archive_revision(source, resource, cache_shard, hashes)
.await?; .await?;
// Before running the build, check that the hashes match. // Before running the build, check that the hashes match.
@ -644,16 +666,12 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>, resource: &PathSourceUrl<'_>,
cache_shard: &CacheShard,
hashes: HashPolicy<'_>, hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> { ) -> Result<ArchiveMetadata, Error> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
);
// Fetch the revision for the source distribution. // Fetch the revision for the source distribution.
let revision = self let revision = self
.archive_revision(source, resource, &cache_shard, hashes) .archive_revision(source, resource, cache_shard, hashes)
.await?; .await?;
// Before running the build, check that the hashes match. // Before running the build, check that the hashes match.

View file

@ -2539,12 +2539,12 @@ fn find_links_offline_no_match() -> Result<()> {
/// Sync using `--find-links` with a local directory. Ensure that cached wheels are reused. /// Sync using `--find-links` with a local directory. Ensure that cached wheels are reused.
#[test] #[test]
fn find_links_cache() -> Result<()> { fn find_links_wheel_cache() -> Result<()> {
let context = TestContext::new("3.12"); let context = TestContext::new("3.12");
let requirements_txt = context.temp_dir.child("requirements.txt"); let requirements_txt = context.temp_dir.child("requirements.txt");
requirements_txt.write_str(indoc! {r" requirements_txt.write_str(indoc! {r"
tqdm tqdm==1000.0.0
"})?; "})?;
// Install `tqdm`. // Install `tqdm`.
@ -2585,6 +2585,55 @@ fn find_links_cache() -> Result<()> {
Ok(()) Ok(())
} }
/// Sync using `--find-links` with a local directory. Ensure that cached source distributions are
/// reused.
#[test]
fn find_links_source_cache() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_txt = context.temp_dir.child("requirements.txt");
requirements_txt.write_str(indoc! {r"
tqdm==999.0.0
"})?;
// Install `tqdm`.
uv_snapshot!(context.filters(), command(&context)
.arg("requirements.txt")
.arg("--find-links")
.arg(context.workspace_root.join("scripts/links/")), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
+ tqdm==999.0.0
"###
);
// Reinstall `tqdm` with `--reinstall`. Ensure that the wheel is reused.
uv_snapshot!(context.filters(), command(&context)
.arg("requirements.txt")
.arg("--reinstall")
.arg("--find-links")
.arg(context.workspace_root.join("scripts/links/")), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Uninstalled 1 package in [TIME]
Installed 1 package in [TIME]
- tqdm==999.0.0
+ tqdm==999.0.0
"###
);
Ok(())
}
/// Install without network access via the `--offline` flag. /// Install without network access via the `--offline` flag.
#[test] #[test]
fn offline() -> Result<()> { fn offline() -> Result<()> {