Wheel metadata refactor (#462)

A consistent cache structure for remote wheel metadata:

 * `<wheel metadata cache>/pypi/foo-1.0.0-py3-none-any.json`
* `<wheel metadata
cache>/<digest(index-url)>/foo-1.0.0-py3-none-any.json`
* `<wheel metadata cache>/url/<digest(url)>/foo-1.0.0-py3-none-any.json`

The source dist caching will use a similar structure (#468).
This commit is contained in:
konsti 2023-11-20 17:26:36 +01:00 committed by GitHub
parent d3e9e1783f
commit f0841cdb6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 153 additions and 47 deletions

View file

@ -157,7 +157,7 @@ impl Graph {
marker: None,
},
Dist::Built(BuiltDist::DirectUrl(wheel)) => Requirement {
name: wheel.name.clone(),
name: wheel.filename.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())),
marker: None,

View file

@ -21,6 +21,7 @@ use distribution_filename::WheelFilename;
use distribution_types::{BuiltDist, Dist, Identifier, Metadata, SourceDist, VersionOrUrl};
use pep508_rs::{MarkerEnvironment, Requirement};
use platform_tags::Tags;
use puffin_cache::metadata::WheelMetadataCache;
use puffin_cache::CanonicalUrl;
use puffin_client::RegistryClient;
use puffin_distribution::Fetcher;
@ -300,8 +301,8 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, Context> {
PubGrubPackage::Package(package_name, _extra, Some(url)) => {
// Emit a request to fetch the metadata for this distribution.
if in_flight.insert_url(url) {
priorities.add(package_name.clone());
let distribution = Dist::from_url(package_name.clone(), url.clone());
priorities.add(distribution.name().clone());
request_sink.unbounded_send(Request::Dist(distribution))?;
}
}
@ -605,26 +606,32 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, Context> {
.await
}
// Fetch wheel metadata from the registry if possible. This is a fast-path to avoid
// reading from the cache in the common case: we cache wheel metadata in the HTTP
// cache, rather than downloading the wheel itself.
Request::Dist(Dist::Built(BuiltDist::Registry(wheel))) => {
let metadata = self
.client
.wheel_metadata(wheel.file.clone())
.map_err(ResolveError::Client)
.await?;
if metadata.name != *wheel.name() {
// Fetch wheel metadata.
Request::Dist(Dist::Built(distribution)) => {
let metadata = match &distribution {
BuiltDist::Registry(wheel) => {
self.client
.wheel_metadata(wheel.index.clone(), wheel.file.clone())
.await?
}
BuiltDist::DirectUrl(wheel) => {
self.client
.wheel_metadata_no_pep658(
&wheel.filename,
&wheel.url,
WheelMetadataCache::Url,
)
.await?
}
};
if metadata.name != *distribution.name() {
return Err(ResolveError::NameMismatch {
metadata: metadata.name,
given: wheel.name().clone(),
given: distribution.name().clone(),
});
}
Ok(Response::Dist(
Dist::Built(BuiltDist::Registry(wheel)),
metadata,
None,
))
Ok(Response::Dist(Dist::Built(distribution), metadata, None))
}
// Fetch distribution metadata.