Avoid batch prefetching for un-optimized registries (#7226)

## Summary

We now track the discovered `IndexCapabilities` for each `IndexUrl`. If
we learn that an index doesn't support range requests, we avoid doing
any batch prefetching.

Closes https://github.com/astral-sh/uv/issues/7221.
This commit is contained in:
Charlie Marsh 2024-09-09 15:46:19 -04:00 committed by GitHub
parent 970bd1aa0c
commit 9a7262c360
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 202 additions and 95 deletions

View file

@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
use tokio::sync::mpsc::Sender;
use tracing::{debug, trace};
use distribution_types::{CompatibleDist, DistributionMetadata};
use distribution_types::{CompatibleDist, DistributionMetadata, IndexCapabilities};
use pep440_rs::Version;
use crate::candidate_selector::CandidateSelector;
@ -52,6 +52,7 @@ impl BatchPrefetcher {
python_requirement: &PythonRequirement,
request_sink: &Sender<Request>,
index: &InMemoryIndex,
capabilities: &IndexCapabilities,
selector: &CandidateSelector,
markers: &ResolverMarkers,
) -> anyhow::Result<(), ResolveError> {
@ -135,8 +136,17 @@ impl BatchPrefetcher {
};
// Avoid prefetching source distributions, which could be expensive.
if !dist.prefetchable() {
let Some(wheel) = dist.wheel() else {
continue;
};
// Avoid prefetching built distributions that don't support _either_ PEP 658 (`.metadata`)
// or range requests.
if !(wheel.file.dist_info_metadata
|| capabilities.supports_range_requests(&wheel.index))
{
debug!("Abandoning prefetch for {wheel} due to missing registry capabilities");
return Ok(());
}
// Avoid prefetching for distributions that don't satisfy the Python requirement.

View file

@ -22,8 +22,8 @@ use tracing::{debug, info, instrument, trace, warn, Level};
use distribution_types::{
BuiltDist, CompatibleDist, Dist, DistributionMetadata, IncompatibleDist, IncompatibleSource,
IncompatibleWheel, IndexLocations, InstalledDist, PythonRequirementKind, RemoteSource,
ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef,
IncompatibleWheel, IndexCapabilities, IndexLocations, InstalledDist, PythonRequirementKind,
RemoteSource, ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef,
};
pub(crate) use fork_map::{ForkMap, ForkSet};
use locals::Locals;
@ -95,6 +95,7 @@ struct ResolverState<InstalledPackages: InstalledPackagesProvider> {
groups: Groups,
preferences: Preferences,
git: GitResolver,
capabilities: IndexCapabilities,
exclusions: Exclusions,
urls: Urls,
locals: Locals,
@ -169,6 +170,7 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider>
python_requirement,
index,
build_context.git(),
build_context.capabilities(),
provider,
installed_packages,
)
@ -187,12 +189,14 @@ impl<Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvider>
python_requirement: &PythonRequirement,
index: &InMemoryIndex,
git: &GitResolver,
capabilities: &IndexCapabilities,
provider: Provider,
installed_packages: InstalledPackages,
) -> Result<Self, ResolveError> {
let state = ResolverState {
index: index.clone(),
git: git.clone(),
capabilities: capabilities.clone(),
selector: CandidateSelector::for_resolution(options, &manifest, &markers),
dependency_mode: options.dependency_mode,
urls: Urls::from_manifest(&manifest, &markers, git, options.dependency_mode)?,
@ -458,6 +462,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
&state.python_requirement,
&request_sink,
&self.index,
&self.capabilities,
&self.selector,
&state.markers,
)?;
@ -1808,7 +1813,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
// Avoid prefetching source distributions with unbounded lower-bound ranges. This
// often leads to failed attempts to build legacy versions of packages that are
// incompatible with modern build tools.
if !dist.prefetchable() {
if dist.wheel().is_some() {
if !self.selector.use_highest_version(&package_name) {
if let Some((lower, _)) = range.iter().next() {
if lower == &Bound::Unbounded {

View file

@ -61,6 +61,7 @@ pub trait ResolverProvider {
dist: &'io Dist,
) -> impl Future<Output = WheelMetadataResult> + 'io;
/// Returns the [`IndexLocations`] used by this resolver.
fn index_locations(&self) -> &IndexLocations;
/// Set the [`uv_distribution::Reporter`] to use for this installer.