mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Return computed hashes from metadata requests (#2951)

Browse source 
## Summary

This PR modifies the distribution database to return both the
`Metadata23` and the computed hashes when clients request metadata.

No behavior changes, but this will be necessary to power
`--generate-hashes`.
This commit is contained in:
Charlie Marsh 2024-04-10 15:31:41 -04:00 • committed by GitHub
parent c18551fd3c
commit 8513d603b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 100 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/uv-distribution/src/distribution_database.rs
View file

@ -25,7 +25,7 @@ use uv_types::BuildContext;
use crate::archive::Archive;
use crate::locks::Locks;
use crate::{Error, LocalWheel, Reporter, SourceDistributionBuilder};
use crate::{ArchiveMetadata, Error, LocalWheel, Reporter, SourceDistributionBuilder};
/// A cached high-level interface to convert distributions (a requirement resolved to a location)
/// to a wheel or wheel metadata.
@ -109,7 +109,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
&self,
dist: &Dist,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
match dist {
Dist::Built(built) => self.get_wheel_metadata(built, hashes).await,
Dist::Source(source) => {
@ -343,16 +343,18 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
&self,
dist: &BuiltDist,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
match self.client.wheel_metadata(dist).boxed().await {
Ok(metadata) => Ok(metadata),
Ok(metadata) => Ok(ArchiveMetadata::from(metadata)),
Err(err) if err.is_http_streaming_unsupported() => {
warn!("Streaming unsupported when fetching metadata for {dist}; downloading wheel directly ({err})");
// If the request failed due to an error that could be resolved by
// downloading the wheel directly, try that.
let wheel = self.get_wheel(dist, hashes).await?;
Ok(wheel.metadata()?)
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
Ok(ArchiveMetadata { metadata, hashes })
}
Err(err) => Err(err.into()),
}
@ -366,7 +368,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
&self,
source: &BuildableSource<'_>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
let no_build = match self.build_context.no_build() {
NoBuild::All => true,
NoBuild::None => false,

19
crates/uv-distribution/src/lib.rs
View file

@ -4,6 +4,7 @@ pub use download::LocalWheel;
pub use error::Error;
pub use git::{is_same_reference, to_precise};
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
use pypi_types::{HashDigest, Metadata23};
pub use reporter::Reporter;
pub use source::SourceDistributionBuilder;
@ -16,3 +17,21 @@ mod index;
mod locks;
mod reporter;
mod source;
/// The metadata associated with an archive.
#[derive(Debug, Clone)]
pub struct ArchiveMetadata {
/// The [`Metadata23`] for the underlying distribution.
pub metadata: Metadata23,
/// The hashes of the source or built archive.
pub hashes: Vec<HashDigest>,
}
impl From<Metadata23> for ArchiveMetadata {
fn from(metadata: Metadata23) -> Self {
Self {
metadata,
hashes: vec![],
}
}
}

54
crates/uv-distribution/src/source/mod.rs
View file

@ -37,7 +37,7 @@ use crate::error::Error;
use crate::git::{fetch_git_archive, resolve_precise};
use crate::source::built_wheel_metadata::BuiltWheelMetadata;
use crate::source::revision::Revision;
use crate::Reporter;
use crate::{ArchiveMetadata, Reporter};
mod built_wheel_metadata;
mod revision;
@ -215,7 +215,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self,
source: &BuildableSource<'_>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
let metadata = match &source {
BuildableSource::Dist(SourceDist::Registry(dist)) => {
let url = match &dist.file.url {
@ -419,7 +419,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
cache_shard: &CacheShard,
subdirectory: Option<&'data Path>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
// Fetch the revision for the source distribution.
let revision = self
.url_revision(source, filename, url, cache_shard, hashes)
@ -442,7 +442,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let metadata_entry = cache_shard.entry(METADATA);
if let Some(metadata) = read_cached_metadata(&metadata_entry).await? {
debug!("Using cached metadata for: {source}");
return Ok(metadata);
return Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
});
}
// Otherwise, we either need to build the metadata or the wheel.
@ -463,7 +466,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
return Ok(metadata);
return Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
});
}
let task = self
@ -488,7 +494,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
}
Ok(metadata)
Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
})
}
/// Return the [`Revision`] for a remote URL, refreshing it if necessary.
@ -632,7 +641,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
@ -660,7 +669,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let metadata_entry = cache_shard.entry(METADATA);
if let Some(metadata) = read_cached_metadata(&metadata_entry).await? {
debug!("Using cached metadata for: {source}");
return Ok(metadata);
return Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
});
}
let source_entry = cache_shard.entry("source");
@ -680,7 +692,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
return Ok(metadata);
return Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
});
}
// Otherwise, we need to build a wheel.
@ -705,7 +720,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
Ok(metadata)
Ok(ArchiveMetadata {
metadata,
hashes: revision.into_hashes(),
})
}
/// Return the [`Revision`] for a local archive, refreshing it if necessary.
@ -826,7 +844,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source: &BuildableSource<'_>,
resource: &PathSourceUrl<'_>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
// Before running the build, check that the hashes match.
if !hashes.is_empty() {
return Err(Error::HashesNotSupportedSourceTree(source.to_string()));
@ -850,7 +868,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let metadata_entry = cache_shard.entry(METADATA);
if let Some(metadata) = read_cached_metadata(&metadata_entry).await? {
debug!("Using cached metadata for: {source}");
return Ok(metadata);
return Ok(ArchiveMetadata::from(metadata));
}
// If the backend supports `prepare_metadata_for_build_wheel`, use it.
@ -868,7 +886,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
return Ok(metadata);
return Ok(ArchiveMetadata::from(metadata));
}
// Otherwise, we need to build a wheel.
@ -893,7 +911,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
Ok(metadata)
Ok(ArchiveMetadata::from(metadata))
}
/// Return the [`Revision`] for a local source tree, refreshing it if necessary.
@ -1000,7 +1018,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source: &BuildableSource<'_>,
resource: &GitSourceUrl<'_>,
hashes: &[HashDigest],
) -> Result<Metadata23, Error> {
) -> Result<ArchiveMetadata, Error> {
// Before running the build, check that the hashes match.
if !hashes.is_empty() {
return Err(Error::HashesNotSupportedGit(source.to_string()));
@ -1039,7 +1057,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
{
if let Some(metadata) = read_cached_metadata(&metadata_entry).await? {
debug!("Using cached metadata for: {source}");
return Ok(metadata);
return Ok(ArchiveMetadata::from(metadata));
}
}
@ -1058,7 +1076,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
return Ok(metadata);
return Ok(ArchiveMetadata::from(metadata));
}
// Otherwise, we need to build a wheel.
@ -1083,7 +1101,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await
.map_err(Error::CacheWrite)?;
Ok(metadata)
Ok(ArchiveMetadata::from(metadata))
}
/// Download and unzip a source distribution into the cache from an HTTP response.

14
crates/uv-requirements/src/lookahead.rs
View file

@ -139,24 +139,24 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> {
// Fetch the metadata for the distribution.
let requires_dist = {
let id = dist.package_id();
if let Some(metadata) = self
if let Some(archive) = self
.index
.get_metadata(&id)
.as_deref()
.and_then(|response| {
if let MetadataResponse::Found(metadata) = response {
Some(metadata)
if let MetadataResponse::Found(archive, ..) = response {
Some(archive)
} else {
None
}
})
{
// If the metadata is already in the index, return it.
metadata.requires_dist.clone()
archive.metadata.requires_dist.clone()
} else {
// Run the PEP 517 build process to extract metadata from the source distribution.
let hashes = self.hashes.get(dist.name()).unwrap_or_default();
let metadata = self
let archive = self
.database
.get_or_build_wheel_metadata(&dist, hashes)
.await
@ -165,11 +165,11 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> {
Dist::Source(source) => format!("Failed to download and build: {source}"),
})?;
let requires_dist = metadata.requires_dist.clone();
let requires_dist = archive.metadata.requires_dist.clone();
// Insert the metadata into the index.
self.index
.insert_metadata(id, MetadataResponse::Found(metadata));
.insert_metadata(id, MetadataResponse::Found(archive));
requires_dist
}

14
crates/uv-requirements/src/source_tree.rs
View file

@ -100,30 +100,30 @@ impl<'a, Context: BuildContext + Send + Sync> SourceTreeResolver<'a, Context> {
// Fetch the metadata for the distribution.
let metadata = {
let id = PackageId::from_url(source.url());
if let Some(metadata) = self
if let Some(archive) = self
.index
.get_metadata(&id)
.as_deref()
.and_then(|response| {
if let MetadataResponse::Found(metadata) = response {
Some(metadata)
if let MetadataResponse::Found(archive) = response {
Some(archive)
} else {
None
}
})
{
// If the metadata is already in the index, return it.
metadata.clone()
archive.metadata.clone()
} else {
// Run the PEP 517 build process to extract metadata from the source distribution.
let source = BuildableSource::Url(source);
let metadata = self.database.build_wheel_metadata(&source, &[]).await?;
let archive = self.database.build_wheel_metadata(&source, &[]).await?;
// Insert the metadata into the index.
self.index
.insert_metadata(id, MetadataResponse::Found(metadata.clone()));
.insert_metadata(id, MetadataResponse::Found(archive.clone()));
metadata
archive.metadata
}
};

14
crates/uv-requirements/src/unnamed.rs
View file

@ -250,24 +250,24 @@ impl<'a, Context: BuildContext + Send + Sync> NamedRequirementsResolver<'a, Cont
// Fetch the metadata for the distribution.
let name = {
let id = PackageId::from_url(source.url());
if let Some(metadata) = index.get_metadata(&id).as_deref().and_then(|response| {
if let MetadataResponse::Found(metadata) = response {
Some(metadata)
if let Some(archive) = index.get_metadata(&id).as_deref().and_then(|response| {
if let MetadataResponse::Found(archive) = response {
Some(archive)
} else {
None
}
}) {
// If the metadata is already in the index, return it.
metadata.name.clone()
archive.metadata.name.clone()
} else {
// Run the PEP 517 build process to extract metadata from the source distribution.
let source = BuildableSource::Url(source);
let metadata = database.build_wheel_metadata(&source, &[]).await?;
let archive = database.build_wheel_metadata(&source, &[]).await?;
let name = metadata.name.clone();
let name = archive.metadata.name.clone();
// Insert the metadata into the index.
index.insert_metadata(id, MetadataResponse::Found(metadata));
index.insert_metadata(id, MetadataResponse::Found(archive));
name
}

12
crates/uv-resolver/src/resolution.rs
View file

@ -177,14 +177,14 @@ impl ResolutionGraph {
)
});
let MetadataResponse::Found(metadata) = &*response else {
let MetadataResponse::Found(archive) = &*response else {
panic!(
"Every package should have metadata: {:?}",
dist.package_id()
)
};
if metadata.provides_extras.contains(extra) {
if archive.metadata.provides_extras.contains(extra) {
extras
.entry(package_name.clone())
.or_insert_with(Vec::new)
@ -231,14 +231,14 @@ impl ResolutionGraph {
)
});
let MetadataResponse::Found(metadata) = &*response else {
let MetadataResponse::Found(archive) = &*response else {
panic!(
"Every package should have metadata: {:?}",
dist.package_id()
)
};
if metadata.provides_extras.contains(extra) {
if archive.metadata.provides_extras.contains(extra) {
extras
.entry(package_name.clone())
.or_insert_with(Vec::new)
@ -441,13 +441,13 @@ impl ResolutionGraph {
.distributions
.get(&package_id)
.expect("every package in resolution graph has metadata");
let MetadataResponse::Found(md) = &*res else {
let MetadataResponse::Found(archive, ..) = &*res else {
panic!(
"Every package should have metadata: {:?}",
dist.package_id()
)
};
for req in manifest.apply(&md.requires_dist) {
for req in manifest.apply(&archive.metadata.requires_dist) {
let Some(ref marker_tree) = req.marker else {
continue;
};

13
crates/uv-resolver/src/resolver/mod.rs
View file

@ -28,7 +28,7 @@ use pypi_types::Metadata23;
pub(crate) use urls::Urls;
use uv_client::RegistryClient;
use uv_configuration::{Constraints, Overrides};
use uv_distribution::DistributionDatabase;
use uv_distribution::{ArchiveMetadata, DistributionDatabase};
use uv_interpreter::Interpreter;
use uv_normalize::PackageName;
use uv_types::{BuildContext, InstalledPackagesProvider, RequiredHashes};
@ -659,7 +659,7 @@ impl<
// If we failed to fetch the metadata for a URL, we can't proceed.
let metadata = match &*response {
MetadataResponse::Found(metadata) => metadata,
MetadataResponse::Found(archive) => &archive.metadata,
MetadataResponse::Offline => {
self.unavailable_packages
.insert(package_name.clone(), UnavailablePackage::Offline);
@ -966,7 +966,7 @@ impl<
.ok_or(ResolveError::Unregistered)?;
let metadata = match &*response {
MetadataResponse::Found(metadata) => metadata,
MetadataResponse::Found(archive) => &archive.metadata,
MetadataResponse::Offline => {
self.incomplete_packages
.entry(package_name.clone())
@ -1067,9 +1067,10 @@ impl<
}
Some(Response::Installed { dist, metadata }) => {
trace!("Received installed distribution metadata for: {dist}");
self.index
.distributions
.done(dist.package_id(), MetadataResponse::Found(metadata));
self.index.distributions.done(
dist.package_id(),
MetadataResponse::Found(ArchiveMetadata::from(metadata)),
);
}
Some(Response::Dist {
dist: Dist::Built(dist),

6
crates/uv-resolver/src/resolver/provider.rs
View file

@ -5,10 +5,10 @@ use chrono::{DateTime, Utc};
use distribution_types::{Dist, IndexLocations, Name};
use platform_tags::Tags;
use pypi_types::Metadata23;
use uv_client::RegistryClient;
use uv_configuration::{NoBinary, NoBuild};
use uv_distribution::DistributionDatabase;
use uv_distribution::{ArchiveMetadata, DistributionDatabase};
use uv_normalize::PackageName;
use uv_types::{BuildContext, RequiredHashes};
@ -36,7 +36,7 @@ pub enum VersionsResponse {
#[derive(Debug)]
pub enum MetadataResponse {
/// The wheel metadata was found and parsed successfully.
Found(Metadata23),
Found(ArchiveMetadata),
/// The wheel metadata was found, but could not be parsed.
InvalidMetadata(Box<pypi_types::MetadataError>),
/// The wheel metadata was found, but the metadata was inconsistent.