mirror of
https://github.com/astral-sh/uv.git
synced 2025-09-27 04:29:10 +00:00
Avoid batch prefetching for un-optimized registries (#7226)
## Summary We now track the discovered `IndexCapabilities` for each `IndexUrl`. If we learn that an index doesn't support range requests, we avoid doing any batch prefetching. Closes https://github.com/astral-sh/uv/issues/7221.
This commit is contained in:
parent
970bd1aa0c
commit
9a7262c360
24 changed files with 202 additions and 95 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
@ -1040,6 +1040,7 @@ dependencies = [
|
||||||
"platform-tags",
|
"platform-tags",
|
||||||
"pypi-types",
|
"pypi-types",
|
||||||
"rkyv",
|
"rkyv",
|
||||||
|
"rustc-hash",
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
|
@ -83,7 +83,7 @@ mod resolver {
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
use distribution_types::IndexLocations;
|
use distribution_types::{IndexCapabilities, IndexLocations};
|
||||||
use install_wheel_rs::linker::LinkMode;
|
use install_wheel_rs::linker::LinkMode;
|
||||||
use pep440_rs::Version;
|
use pep440_rs::Version;
|
||||||
use pep508_rs::{MarkerEnvironment, MarkerEnvironmentBuilder};
|
use pep508_rs::{MarkerEnvironment, MarkerEnvironmentBuilder};
|
||||||
|
@ -152,6 +152,7 @@ mod resolver {
|
||||||
);
|
);
|
||||||
let flat_index = FlatIndex::default();
|
let flat_index = FlatIndex::default();
|
||||||
let git = GitResolver::default();
|
let git = GitResolver::default();
|
||||||
|
let capabilities = IndexCapabilities::default();
|
||||||
let hashes = HashStrategy::None;
|
let hashes = HashStrategy::None;
|
||||||
let in_flight = InFlight::default();
|
let in_flight = InFlight::default();
|
||||||
let index = InMemoryIndex::default();
|
let index = InMemoryIndex::default();
|
||||||
|
@ -179,6 +180,7 @@ mod resolver {
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&index,
|
&index,
|
||||||
&git,
|
&git,
|
||||||
|
&capabilities,
|
||||||
&in_flight,
|
&in_flight,
|
||||||
IndexStrategy::default(),
|
IndexStrategy::default(),
|
||||||
&config_settings,
|
&config_settings,
|
||||||
|
|
|
@ -28,6 +28,7 @@ fs-err = { workspace = true }
|
||||||
itertools = { workspace = true }
|
itertools = { workspace = true }
|
||||||
jiff = { workspace = true }
|
jiff = { workspace = true }
|
||||||
rkyv = { workspace = true }
|
rkyv = { workspace = true }
|
||||||
|
rustc-hash = { workspace = true }
|
||||||
schemars = { workspace = true, optional = true }
|
schemars = { workspace = true, optional = true }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|
|
@ -1,10 +1,11 @@
|
||||||
use itertools::Either;
|
use itertools::Either;
|
||||||
|
use rustc_hash::FxHashSet;
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::fmt::{Display, Formatter};
|
use std::fmt::{Display, Formatter};
|
||||||
use std::ops::Deref;
|
use std::ops::Deref;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::sync::LazyLock;
|
use std::sync::{Arc, LazyLock, RwLock};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use url::{ParseError, Url};
|
use url::{ParseError, Url};
|
||||||
|
|
||||||
|
@ -485,3 +486,27 @@ impl From<IndexLocations> for IndexUrls {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A map of [`IndexUrl`]s to their capabilities.
|
||||||
|
///
|
||||||
|
/// For now, we only support a single capability (range requests), and we only store an index if
|
||||||
|
/// it _doesn't_ support range requests. The benefit is that the map is almost always empty, so
|
||||||
|
/// validating capabilities is extremely cheap.
|
||||||
|
#[derive(Debug, Default, Clone)]
|
||||||
|
pub struct IndexCapabilities(Arc<RwLock<FxHashSet<IndexUrl>>>);
|
||||||
|
|
||||||
|
impl IndexCapabilities {
|
||||||
|
/// Returns `true` if the given [`IndexUrl`] supports range requests.
|
||||||
|
pub fn supports_range_requests(&self, index_url: &IndexUrl) -> bool {
|
||||||
|
!self.0.read().unwrap().contains(index_url)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark an [`IndexUrl`] as not supporting range requests.
|
||||||
|
pub fn set_supports_range_requests(&self, index_url: IndexUrl, supports: bool) {
|
||||||
|
if supports {
|
||||||
|
self.0.write().unwrap().remove(&index_url);
|
||||||
|
} else {
|
||||||
|
self.0.write().unwrap().insert(index_url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -468,15 +468,14 @@ impl<'a> CompatibleDist<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns whether the distribution is a source distribution.
|
/// Returns a [`RegistryBuiltWheel`] if the distribution includes a compatible or incompatible
|
||||||
///
|
/// wheel.
|
||||||
/// Avoid building source distributions we don't need.
|
pub fn wheel(&self) -> Option<&RegistryBuiltWheel> {
|
||||||
pub fn prefetchable(&self) -> bool {
|
match self {
|
||||||
match *self {
|
CompatibleDist::InstalledDist(_) => None,
|
||||||
CompatibleDist::SourceDist { .. } => false,
|
CompatibleDist::SourceDist { .. } => None,
|
||||||
CompatibleDist::InstalledDist(_)
|
CompatibleDist::CompatibleWheel { wheel, .. } => Some(wheel),
|
||||||
| CompatibleDist::CompatibleWheel { .. }
|
CompatibleDist::IncompatibleWheel { wheel, .. } => Some(wheel),
|
||||||
| CompatibleDist::IncompatibleWheel { .. } => true,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
use std::io;
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
|
@ -16,7 +15,9 @@ use tracing::{info_span, instrument, trace, warn, Instrument};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
|
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
|
||||||
use distribution_types::{BuiltDist, File, FileLocation, IndexUrl, IndexUrls, Name};
|
use distribution_types::{
|
||||||
|
BuiltDist, File, FileLocation, IndexCapabilities, IndexUrl, IndexUrls, Name,
|
||||||
|
};
|
||||||
use install_wheel_rs::metadata::{find_archive_dist_info, is_metadata_entry};
|
use install_wheel_rs::metadata::{find_archive_dist_info, is_metadata_entry};
|
||||||
use pep440_rs::Version;
|
use pep440_rs::Version;
|
||||||
use pep508_rs::MarkerEnvironment;
|
use pep508_rs::MarkerEnvironment;
|
||||||
|
@ -147,7 +148,7 @@ impl<'a> RegistryClientBuilder<'a> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> TryFrom<BaseClientBuilder<'a>> for RegistryClientBuilder<'a> {
|
impl<'a> TryFrom<BaseClientBuilder<'a>> for RegistryClientBuilder<'a> {
|
||||||
type Error = io::Error;
|
type Error = std::io::Error;
|
||||||
|
|
||||||
fn try_from(value: BaseClientBuilder<'a>) -> Result<Self, Self::Error> {
|
fn try_from(value: BaseClientBuilder<'a>) -> Result<Self, Self::Error> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
@ -402,7 +403,11 @@ impl RegistryClient {
|
||||||
/// 2. From a remote wheel by partial zip reading
|
/// 2. From a remote wheel by partial zip reading
|
||||||
/// 3. From a (temp) download of a remote wheel (this is a fallback, the webserver should support range requests)
|
/// 3. From a (temp) download of a remote wheel (this is a fallback, the webserver should support range requests)
|
||||||
#[instrument(skip_all, fields(% built_dist))]
|
#[instrument(skip_all, fields(% built_dist))]
|
||||||
pub async fn wheel_metadata(&self, built_dist: &BuiltDist) -> Result<Metadata23, Error> {
|
pub async fn wheel_metadata(
|
||||||
|
&self,
|
||||||
|
built_dist: &BuiltDist,
|
||||||
|
capabilities: &IndexCapabilities,
|
||||||
|
) -> Result<Metadata23, Error> {
|
||||||
let metadata = match &built_dist {
|
let metadata = match &built_dist {
|
||||||
BuiltDist::Registry(wheels) => {
|
BuiltDist::Registry(wheels) => {
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
@ -451,7 +456,7 @@ impl RegistryClient {
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
WheelLocation::Url(url) => {
|
WheelLocation::Url(url) => {
|
||||||
self.wheel_metadata_registry(&wheel.index, &wheel.file, &url)
|
self.wheel_metadata_registry(&wheel.index, &wheel.file, &url, capabilities)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -460,7 +465,9 @@ impl RegistryClient {
|
||||||
self.wheel_metadata_no_pep658(
|
self.wheel_metadata_no_pep658(
|
||||||
&wheel.filename,
|
&wheel.filename,
|
||||||
&wheel.url,
|
&wheel.url,
|
||||||
|
None,
|
||||||
WheelCache::Url(&wheel.url),
|
WheelCache::Url(&wheel.url),
|
||||||
|
capabilities,
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
|
@ -489,6 +496,7 @@ impl RegistryClient {
|
||||||
index: &IndexUrl,
|
index: &IndexUrl,
|
||||||
file: &File,
|
file: &File,
|
||||||
url: &Url,
|
url: &Url,
|
||||||
|
capabilities: &IndexCapabilities,
|
||||||
) -> Result<Metadata23, Error> {
|
) -> Result<Metadata23, Error> {
|
||||||
// If the metadata file is available at its own url (PEP 658), download it from there.
|
// If the metadata file is available at its own url (PEP 658), download it from there.
|
||||||
let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?;
|
let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?;
|
||||||
|
@ -536,8 +544,14 @@ impl RegistryClient {
|
||||||
// If we lack PEP 658 support, try using HTTP range requests to read only the
|
// If we lack PEP 658 support, try using HTTP range requests to read only the
|
||||||
// `.dist-info/METADATA` file from the zip, and if that also fails, download the whole wheel
|
// `.dist-info/METADATA` file from the zip, and if that also fails, download the whole wheel
|
||||||
// into the cache and read from there
|
// into the cache and read from there
|
||||||
self.wheel_metadata_no_pep658(&filename, url, WheelCache::Index(index))
|
self.wheel_metadata_no_pep658(
|
||||||
.await
|
&filename,
|
||||||
|
url,
|
||||||
|
Some(index),
|
||||||
|
WheelCache::Index(index),
|
||||||
|
capabilities,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -546,7 +560,9 @@ impl RegistryClient {
|
||||||
&self,
|
&self,
|
||||||
filename: &'data WheelFilename,
|
filename: &'data WheelFilename,
|
||||||
url: &'data Url,
|
url: &'data Url,
|
||||||
|
index: Option<&'data IndexUrl>,
|
||||||
cache_shard: WheelCache<'data>,
|
cache_shard: WheelCache<'data>,
|
||||||
|
capabilities: &'data IndexCapabilities,
|
||||||
) -> Result<Metadata23, Error> {
|
) -> Result<Metadata23, Error> {
|
||||||
let cache_entry = self.cache.entry(
|
let cache_entry = self.cache.entry(
|
||||||
CacheBucket::Wheels,
|
CacheBucket::Wheels,
|
||||||
|
@ -562,72 +578,80 @@ impl RegistryClient {
|
||||||
Connectivity::Offline => CacheControl::AllowStale,
|
Connectivity::Offline => CacheControl::AllowStale,
|
||||||
};
|
};
|
||||||
|
|
||||||
let req = self
|
// Attempt to fetch via a range request.
|
||||||
.uncached_client(url)
|
if index.map_or(true, |index| capabilities.supports_range_requests(index)) {
|
||||||
.head(url.clone())
|
let req = self
|
||||||
.header(
|
.uncached_client(url)
|
||||||
"accept-encoding",
|
.head(url.clone())
|
||||||
http::HeaderValue::from_static("identity"),
|
.header(
|
||||||
)
|
"accept-encoding",
|
||||||
.build()
|
http::HeaderValue::from_static("identity"),
|
||||||
.map_err(ErrorKind::from)?;
|
)
|
||||||
|
.build()
|
||||||
|
.map_err(ErrorKind::from)?;
|
||||||
|
|
||||||
// Copy authorization headers from the HEAD request to subsequent requests
|
// Copy authorization headers from the HEAD request to subsequent requests
|
||||||
let mut headers = HeaderMap::default();
|
let mut headers = HeaderMap::default();
|
||||||
if let Some(authorization) = req.headers().get("authorization") {
|
if let Some(authorization) = req.headers().get("authorization") {
|
||||||
headers.append("authorization", authorization.clone());
|
headers.append("authorization", authorization.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
// This response callback is special, we actually make a number of subsequent requests to
|
// This response callback is special, we actually make a number of subsequent requests to
|
||||||
// fetch the file from the remote zip.
|
// fetch the file from the remote zip.
|
||||||
let read_metadata_range_request = |response: Response| {
|
let read_metadata_range_request = |response: Response| {
|
||||||
async {
|
async {
|
||||||
let mut reader = AsyncHttpRangeReader::from_head_response(
|
let mut reader = AsyncHttpRangeReader::from_head_response(
|
||||||
self.uncached_client(url).clone(),
|
self.uncached_client(url).clone(),
|
||||||
response,
|
response,
|
||||||
url.clone(),
|
url.clone(),
|
||||||
headers,
|
headers,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(ErrorKind::AsyncHttpRangeReader)?;
|
||||||
|
trace!("Getting metadata for {filename} by range request");
|
||||||
|
let text = wheel_metadata_from_remote_zip(filename, &mut reader).await?;
|
||||||
|
let metadata = Metadata23::parse_metadata(text.as_bytes()).map_err(|err| {
|
||||||
|
Error::from(ErrorKind::MetadataParseError(
|
||||||
|
filename.clone(),
|
||||||
|
url.to_string(),
|
||||||
|
Box::new(err),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
Ok::<Metadata23, CachedClientError<Error>>(metadata)
|
||||||
|
}
|
||||||
|
.boxed_local()
|
||||||
|
.instrument(info_span!("read_metadata_range_request", wheel = %filename))
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = self
|
||||||
|
.cached_client()
|
||||||
|
.get_serde(
|
||||||
|
req,
|
||||||
|
&cache_entry,
|
||||||
|
cache_control,
|
||||||
|
read_metadata_range_request,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(ErrorKind::AsyncHttpRangeReader)?;
|
.map_err(crate::Error::from);
|
||||||
trace!("Getting metadata for {filename} by range request");
|
|
||||||
let text = wheel_metadata_from_remote_zip(filename, &mut reader).await?;
|
|
||||||
let metadata = Metadata23::parse_metadata(text.as_bytes()).map_err(|err| {
|
|
||||||
Error::from(ErrorKind::MetadataParseError(
|
|
||||||
filename.clone(),
|
|
||||||
url.to_string(),
|
|
||||||
Box::new(err),
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
Ok::<Metadata23, CachedClientError<Error>>(metadata)
|
|
||||||
}
|
|
||||||
.boxed_local()
|
|
||||||
.instrument(info_span!("read_metadata_range_request", wheel = %filename))
|
|
||||||
};
|
|
||||||
|
|
||||||
let result = self
|
match result {
|
||||||
.cached_client()
|
Ok(metadata) => return Ok(metadata),
|
||||||
.get_serde(
|
Err(err) => {
|
||||||
req,
|
if err.is_http_range_requests_unsupported() {
|
||||||
&cache_entry,
|
// The range request version failed. Fall back to streaming the file to search
|
||||||
cache_control,
|
// for the METADATA file.
|
||||||
read_metadata_range_request,
|
warn!("Range requests not supported for {filename}; streaming wheel");
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(crate::Error::from);
|
|
||||||
|
|
||||||
match result {
|
// Mark the index as not supporting range requests.
|
||||||
Ok(metadata) => return Ok(metadata),
|
if let Some(index) = index {
|
||||||
Err(err) => {
|
capabilities.set_supports_range_requests(index.clone(), false);
|
||||||
if err.is_http_range_requests_unsupported() {
|
}
|
||||||
// The range request version failed. Fall back to streaming the file to search
|
} else {
|
||||||
// for the METADATA file.
|
return Err(err);
|
||||||
warn!("Range requests not supported for {filename}; streaming wheel");
|
}
|
||||||
} else {
|
|
||||||
return Err(err);
|
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
};
|
}
|
||||||
|
|
||||||
// Create a request to stream the file.
|
// Create a request to stream the file.
|
||||||
let req = self
|
let req = self
|
||||||
|
|
|
@ -4,7 +4,7 @@ use anyhow::Result;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use distribution_filename::WheelFilename;
|
use distribution_filename::WheelFilename;
|
||||||
use distribution_types::{BuiltDist, DirectUrlBuiltDist};
|
use distribution_types::{BuiltDist, DirectUrlBuiltDist, IndexCapabilities};
|
||||||
use pep508_rs::VerbatimUrl;
|
use pep508_rs::VerbatimUrl;
|
||||||
use uv_cache::Cache;
|
use uv_cache::Cache;
|
||||||
use uv_client::RegistryClientBuilder;
|
use uv_client::RegistryClientBuilder;
|
||||||
|
@ -24,7 +24,8 @@ async fn remote_metadata_with_and_without_cache() -> Result<()> {
|
||||||
location: Url::parse(url).unwrap(),
|
location: Url::parse(url).unwrap(),
|
||||||
url: VerbatimUrl::from_str(url).unwrap(),
|
url: VerbatimUrl::from_str(url).unwrap(),
|
||||||
});
|
});
|
||||||
let metadata = client.wheel_metadata(&dist).await.unwrap();
|
let capabilities = IndexCapabilities::default();
|
||||||
|
let metadata = client.wheel_metadata(&dist, &capabilities).await.unwrap();
|
||||||
assert_eq!(metadata.version.to_string(), "4.66.1");
|
assert_eq!(metadata.version.to_string(), "4.66.1");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ use anyhow::{bail, Result};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
|
||||||
use distribution_filename::WheelFilename;
|
use distribution_filename::WheelFilename;
|
||||||
use distribution_types::{BuiltDist, DirectUrlBuiltDist, RemoteSource};
|
use distribution_types::{BuiltDist, DirectUrlBuiltDist, IndexCapabilities, RemoteSource};
|
||||||
use pep508_rs::VerbatimUrl;
|
use pep508_rs::VerbatimUrl;
|
||||||
use pypi_types::ParsedUrl;
|
use pypi_types::ParsedUrl;
|
||||||
use uv_cache::{Cache, CacheArgs};
|
use uv_cache::{Cache, CacheArgs};
|
||||||
|
@ -21,6 +21,7 @@ pub(crate) struct WheelMetadataArgs {
|
||||||
pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> Result<()> {
|
pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> Result<()> {
|
||||||
let cache = Cache::try_from(args.cache_args)?.init()?;
|
let cache = Cache::try_from(args.cache_args)?.init()?;
|
||||||
let client = RegistryClientBuilder::new(cache).build();
|
let client = RegistryClientBuilder::new(cache).build();
|
||||||
|
let capabilities = IndexCapabilities::default();
|
||||||
|
|
||||||
let filename = WheelFilename::from_str(&args.url.filename()?)?;
|
let filename = WheelFilename::from_str(&args.url.filename()?)?;
|
||||||
|
|
||||||
|
@ -29,11 +30,14 @@ pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> Result<()> {
|
||||||
};
|
};
|
||||||
|
|
||||||
let metadata = client
|
let metadata = client
|
||||||
.wheel_metadata(&BuiltDist::DirectUrl(DirectUrlBuiltDist {
|
.wheel_metadata(
|
||||||
filename,
|
&BuiltDist::DirectUrl(DirectUrlBuiltDist {
|
||||||
location: archive.url,
|
filename,
|
||||||
url: args.url,
|
location: archive.url,
|
||||||
}))
|
url: args.url,
|
||||||
|
}),
|
||||||
|
&capabilities,
|
||||||
|
)
|
||||||
.await?;
|
.await?;
|
||||||
println!("{metadata:?}");
|
println!("{metadata:?}");
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
|
@ -11,7 +11,9 @@ use itertools::Itertools;
|
||||||
use rustc_hash::FxHashMap;
|
use rustc_hash::FxHashMap;
|
||||||
use tracing::{debug, instrument};
|
use tracing::{debug, instrument};
|
||||||
|
|
||||||
use distribution_types::{CachedDist, IndexLocations, Name, Resolution, SourceDist};
|
use distribution_types::{
|
||||||
|
CachedDist, IndexCapabilities, IndexLocations, Name, Resolution, SourceDist,
|
||||||
|
};
|
||||||
use pypi_types::Requirement;
|
use pypi_types::Requirement;
|
||||||
use uv_build::{SourceBuild, SourceBuildContext};
|
use uv_build::{SourceBuild, SourceBuildContext};
|
||||||
use uv_cache::Cache;
|
use uv_cache::Cache;
|
||||||
|
@ -42,6 +44,7 @@ pub struct BuildDispatch<'a> {
|
||||||
flat_index: &'a FlatIndex,
|
flat_index: &'a FlatIndex,
|
||||||
index: &'a InMemoryIndex,
|
index: &'a InMemoryIndex,
|
||||||
git: &'a GitResolver,
|
git: &'a GitResolver,
|
||||||
|
capabilities: &'a IndexCapabilities,
|
||||||
in_flight: &'a InFlight,
|
in_flight: &'a InFlight,
|
||||||
build_isolation: BuildIsolation<'a>,
|
build_isolation: BuildIsolation<'a>,
|
||||||
link_mode: install_wheel_rs::linker::LinkMode,
|
link_mode: install_wheel_rs::linker::LinkMode,
|
||||||
|
@ -65,6 +68,7 @@ impl<'a> BuildDispatch<'a> {
|
||||||
flat_index: &'a FlatIndex,
|
flat_index: &'a FlatIndex,
|
||||||
index: &'a InMemoryIndex,
|
index: &'a InMemoryIndex,
|
||||||
git: &'a GitResolver,
|
git: &'a GitResolver,
|
||||||
|
capabilities: &'a IndexCapabilities,
|
||||||
in_flight: &'a InFlight,
|
in_flight: &'a InFlight,
|
||||||
index_strategy: IndexStrategy,
|
index_strategy: IndexStrategy,
|
||||||
config_settings: &'a ConfigSettings,
|
config_settings: &'a ConfigSettings,
|
||||||
|
@ -85,6 +89,7 @@ impl<'a> BuildDispatch<'a> {
|
||||||
flat_index,
|
flat_index,
|
||||||
index,
|
index,
|
||||||
git,
|
git,
|
||||||
|
capabilities,
|
||||||
in_flight,
|
in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_settings,
|
config_settings,
|
||||||
|
@ -127,6 +132,10 @@ impl<'a> BuildContext for BuildDispatch<'a> {
|
||||||
self.git
|
self.git
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn capabilities(&self) -> &IndexCapabilities {
|
||||||
|
self.capabilities
|
||||||
|
}
|
||||||
|
|
||||||
fn build_options(&self) -> &BuildOptions {
|
fn build_options(&self) -> &BuildOptions {
|
||||||
self.build_options
|
self.build_options
|
||||||
}
|
}
|
||||||
|
|
|
@ -373,7 +373,11 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
|
||||||
|
|
||||||
let result = self
|
let result = self
|
||||||
.client
|
.client
|
||||||
.managed(|client| client.wheel_metadata(dist).boxed_local())
|
.managed(|client| {
|
||||||
|
client
|
||||||
|
.wheel_metadata(dist, self.build_context.capabilities())
|
||||||
|
.boxed_local()
|
||||||
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
|
|
|
@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
|
||||||
use tokio::sync::mpsc::Sender;
|
use tokio::sync::mpsc::Sender;
|
||||||
use tracing::{debug, trace};
|
use tracing::{debug, trace};
|
||||||
|
|
||||||
use distribution_types::{CompatibleDist, DistributionMetadata};
|
use distribution_types::{CompatibleDist, DistributionMetadata, IndexCapabilities};
|
||||||
use pep440_rs::Version;
|
use pep440_rs::Version;
|
||||||
|
|
||||||
use crate::candidate_selector::CandidateSelector;
|
use crate::candidate_selector::CandidateSelector;
|
||||||
|
@ -52,6 +52,7 @@ impl BatchPrefetcher {
|
||||||
python_requirement: &PythonRequirement,
|
python_requirement: &PythonRequirement,
|
||||||
request_sink: &Sender<Request>,
|
request_sink: &Sender<Request>,
|
||||||
index: &InMemoryIndex,
|
index: &InMemoryIndex,
|
||||||
|
capabilities: &IndexCapabilities,
|
||||||
selector: &CandidateSelector,
|
selector: &CandidateSelector,
|
||||||
markers: &ResolverMarkers,
|
markers: &ResolverMarkers,
|
||||||
) -> anyhow::Result<(), ResolveError> {
|
) -> anyhow::Result<(), ResolveError> {
|
||||||
|
@ -135,8 +136,17 @@ impl BatchPrefetcher {
|
||||||
};
|
};
|
||||||
|
|
||||||
// Avoid prefetching source distributions, which could be expensive.
|
// Avoid prefetching source distributions, which could be expensive.
|
||||||
if !dist.prefetchable() {
|
let Some(wheel) = dist.wheel() else {
|
||||||
continue;
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Avoid prefetching built distributions that don't support _either_ PEP 658 (`.metadata`)
|
||||||
|
// or range requests.
|
||||||
|
if !(wheel.file.dist_info_metadata
|
||||||
|
|| capabilities.supports_range_requests(&wheel.index))
|
||||||
|
{
|
||||||
|
debug!("Abandoning prefetch for {wheel} due to missing registry capabilities");
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Avoid prefetching for distributions that don't satisfy the Python requirement.
|
// Avoid prefetching for distributions that don't satisfy the Python requirement.
|
||||||
|
|
|
@ -22,8 +22,8 @@ use tracing::{debug, info, instrument, trace, warn, Level};
|
||||||
|
|
||||||
use distribution_types::{
|
use distribution_types::{
|
||||||
BuiltDist, CompatibleDist, Dist, DistributionMetadata, IncompatibleDist, IncompatibleSource,
|
BuiltDist, CompatibleDist, Dist, DistributionMetadata, IncompatibleDist, IncompatibleSource,
|
||||||
IncompatibleWheel, IndexLocations, InstalledDist, PythonRequirementKind, RemoteSource,
|
IncompatibleWheel, IndexCapabilities, IndexLocations, InstalledDist, PythonRequirementKind,
|
||||||
ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef,
|
RemoteSource, ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef,
|
||||||
};
|
};
|
||||||
pub(crate) use fork_map::{ForkMap, ForkSet};
|
pub(crate) use fork_map::{ForkMap, ForkSet};
|
||||||
use locals::Locals;
|
use locals::Locals;
|
||||||
|
@ -95,6 +95,7 @@ struct ResolverState<InstalledPackages: InstalledPackagesProvider> {
|
||||||
groups: Groups,
|
groups: Groups,
|
||||||
preferences: Preferences,
|
preferences: Preferences,
|
||||||
git: GitResolver,
|
git: GitResolver,
|
||||||
|
capabilities: IndexCapabilities,
|
||||||
exclusions: Exclusions,
|
exclusions: Exclusions,
|
||||||
urls: Urls,
|
urls: Urls,
|
||||||
locals: Locals,
|
locals: Locals,
|
||||||
|
@ -169,6 +170,7 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider>
|
||||||
python_requirement,
|
python_requirement,
|
||||||
index,
|
index,
|
||||||
build_context.git(),
|
build_context.git(),
|
||||||
|
build_context.capabilities(),
|
||||||
provider,
|
provider,
|
||||||
installed_packages,
|
installed_packages,
|
||||||
)
|
)
|
||||||
|
@ -187,12 +189,14 @@ impl<Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvider>
|
||||||
python_requirement: &PythonRequirement,
|
python_requirement: &PythonRequirement,
|
||||||
index: &InMemoryIndex,
|
index: &InMemoryIndex,
|
||||||
git: &GitResolver,
|
git: &GitResolver,
|
||||||
|
capabilities: &IndexCapabilities,
|
||||||
provider: Provider,
|
provider: Provider,
|
||||||
installed_packages: InstalledPackages,
|
installed_packages: InstalledPackages,
|
||||||
) -> Result<Self, ResolveError> {
|
) -> Result<Self, ResolveError> {
|
||||||
let state = ResolverState {
|
let state = ResolverState {
|
||||||
index: index.clone(),
|
index: index.clone(),
|
||||||
git: git.clone(),
|
git: git.clone(),
|
||||||
|
capabilities: capabilities.clone(),
|
||||||
selector: CandidateSelector::for_resolution(options, &manifest, &markers),
|
selector: CandidateSelector::for_resolution(options, &manifest, &markers),
|
||||||
dependency_mode: options.dependency_mode,
|
dependency_mode: options.dependency_mode,
|
||||||
urls: Urls::from_manifest(&manifest, &markers, git, options.dependency_mode)?,
|
urls: Urls::from_manifest(&manifest, &markers, git, options.dependency_mode)?,
|
||||||
|
@ -458,6 +462,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
||||||
&state.python_requirement,
|
&state.python_requirement,
|
||||||
&request_sink,
|
&request_sink,
|
||||||
&self.index,
|
&self.index,
|
||||||
|
&self.capabilities,
|
||||||
&self.selector,
|
&self.selector,
|
||||||
&state.markers,
|
&state.markers,
|
||||||
)?;
|
)?;
|
||||||
|
@ -1808,7 +1813,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
||||||
// Avoid prefetching source distributions with unbounded lower-bound ranges. This
|
// Avoid prefetching source distributions with unbounded lower-bound ranges. This
|
||||||
// often leads to failed attempts to build legacy versions of packages that are
|
// often leads to failed attempts to build legacy versions of packages that are
|
||||||
// incompatible with modern build tools.
|
// incompatible with modern build tools.
|
||||||
if !dist.prefetchable() {
|
if dist.wheel().is_some() {
|
||||||
if !self.selector.use_highest_version(&package_name) {
|
if !self.selector.use_highest_version(&package_name) {
|
||||||
if let Some((lower, _)) = range.iter().next() {
|
if let Some((lower, _)) = range.iter().next() {
|
||||||
if lower == &Bound::Unbounded {
|
if lower == &Bound::Unbounded {
|
||||||
|
|
|
@ -61,6 +61,7 @@ pub trait ResolverProvider {
|
||||||
dist: &'io Dist,
|
dist: &'io Dist,
|
||||||
) -> impl Future<Output = WheelMetadataResult> + 'io;
|
) -> impl Future<Output = WheelMetadataResult> + 'io;
|
||||||
|
|
||||||
|
/// Returns the [`IndexLocations`] used by this resolver.
|
||||||
fn index_locations(&self) -> &IndexLocations;
|
fn index_locations(&self) -> &IndexLocations;
|
||||||
|
|
||||||
/// Set the [`uv_distribution::Reporter`] to use for this installer.
|
/// Set the [`uv_distribution::Reporter`] to use for this installer.
|
||||||
|
|
|
@ -3,7 +3,9 @@ use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
use distribution_types::{CachedDist, IndexLocations, InstalledDist, Resolution, SourceDist};
|
use distribution_types::{
|
||||||
|
CachedDist, IndexCapabilities, IndexLocations, InstalledDist, Resolution, SourceDist,
|
||||||
|
};
|
||||||
use pep508_rs::PackageName;
|
use pep508_rs::PackageName;
|
||||||
use pypi_types::Requirement;
|
use pypi_types::Requirement;
|
||||||
use uv_cache::Cache;
|
use uv_cache::Cache;
|
||||||
|
@ -57,6 +59,9 @@ pub trait BuildContext {
|
||||||
/// Return a reference to the Git resolver.
|
/// Return a reference to the Git resolver.
|
||||||
fn git(&self) -> &GitResolver;
|
fn git(&self) -> &GitResolver;
|
||||||
|
|
||||||
|
/// Return a reference to the discovered registry capabilities.
|
||||||
|
fn capabilities(&self) -> &IndexCapabilities;
|
||||||
|
|
||||||
/// Whether source distribution building or pre-built wheels is disabled.
|
/// Whether source distribution building or pre-built wheels is disabled.
|
||||||
///
|
///
|
||||||
/// This [`BuildContext::setup_build`] calls will fail if builds are disabled.
|
/// This [`BuildContext::setup_build`] calls will fail if builds are disabled.
|
||||||
|
|
|
@ -301,6 +301,7 @@ async fn build_impl(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
|
|
@ -11,7 +11,7 @@ pub(crate) use build::build;
|
||||||
pub(crate) use cache_clean::cache_clean;
|
pub(crate) use cache_clean::cache_clean;
|
||||||
pub(crate) use cache_dir::cache_dir;
|
pub(crate) use cache_dir::cache_dir;
|
||||||
pub(crate) use cache_prune::cache_prune;
|
pub(crate) use cache_prune::cache_prune;
|
||||||
use distribution_types::InstalledMetadata;
|
use distribution_types::{IndexCapabilities, InstalledMetadata};
|
||||||
pub(crate) use help::help;
|
pub(crate) use help::help;
|
||||||
pub(crate) use pip::check::pip_check;
|
pub(crate) use pip::check::pip_check;
|
||||||
pub(crate) use pip::compile::pip_compile;
|
pub(crate) use pip::compile::pip_compile;
|
||||||
|
@ -200,6 +200,8 @@ pub(crate) struct SharedState {
|
||||||
pub(crate) index: InMemoryIndex,
|
pub(crate) index: InMemoryIndex,
|
||||||
/// The downloaded distributions.
|
/// The downloaded distributions.
|
||||||
pub(crate) in_flight: InFlight,
|
pub(crate) in_flight: InFlight,
|
||||||
|
/// The discovered capabilities for each registry index.
|
||||||
|
pub(crate) capabilities: IndexCapabilities,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A multicasting writer that writes to both the standard output and an output file, if present.
|
/// A multicasting writer that writes to both the standard output and an output file, if present.
|
||||||
|
|
|
@ -8,7 +8,8 @@ use owo_colors::OwoColorize;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
use distribution_types::{
|
use distribution_types::{
|
||||||
IndexLocations, NameRequirementSpecification, UnresolvedRequirementSpecification, Verbatim,
|
IndexCapabilities, IndexLocations, NameRequirementSpecification,
|
||||||
|
UnresolvedRequirementSpecification, Verbatim,
|
||||||
};
|
};
|
||||||
use install_wheel_rs::linker::LinkMode;
|
use install_wheel_rs::linker::LinkMode;
|
||||||
use pypi_types::{Requirement, SupportedEnvironments};
|
use pypi_types::{Requirement, SupportedEnvironments};
|
||||||
|
@ -292,6 +293,7 @@ pub(crate) async fn pip_compile(
|
||||||
// Read the lockfile, if present.
|
// Read the lockfile, if present.
|
||||||
let preferences = read_requirements_txt(output_file, &upgrade).await?;
|
let preferences = read_requirements_txt(output_file, &upgrade).await?;
|
||||||
let git = GitResolver::default();
|
let git = GitResolver::default();
|
||||||
|
let capabilities = IndexCapabilities::default();
|
||||||
|
|
||||||
// Combine the `--no-binary` and `--no-build` flags from the requirements files.
|
// Combine the `--no-binary` and `--no-build` flags from the requirements files.
|
||||||
let build_options = build_options.combine(no_binary, no_build);
|
let build_options = build_options.combine(no_binary, no_build);
|
||||||
|
@ -335,6 +337,7 @@ pub(crate) async fn pip_compile(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&source_index,
|
&source_index,
|
||||||
&git,
|
&git,
|
||||||
|
&capabilities,
|
||||||
&in_flight,
|
&in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
&config_settings,
|
&config_settings,
|
||||||
|
|
|
@ -339,6 +339,7 @@ pub(crate) async fn pip_install(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_settings,
|
config_settings,
|
||||||
|
|
|
@ -289,6 +289,7 @@ pub(crate) async fn pip_sync(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_settings,
|
config_settings,
|
||||||
|
|
|
@ -299,6 +299,7 @@ pub(crate) async fn add(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
settings.index_strategy,
|
settings.index_strategy,
|
||||||
&settings.config_setting,
|
&settings.config_setting,
|
||||||
|
|
|
@ -411,6 +411,7 @@ async fn do_lock(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
|
|
@ -519,6 +519,7 @@ pub(crate) async fn resolve_names(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
*index_strategy,
|
*index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
@ -657,6 +658,7 @@ pub(crate) async fn resolve_environment<'a>(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
@ -785,6 +787,7 @@ pub(crate) async fn sync_environment(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
@ -984,6 +987,7 @@ pub(crate) async fn update_environment(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
*index_strategy,
|
*index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
|
|
@ -280,6 +280,7 @@ pub(super) async fn do_sync(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
config_setting,
|
config_setting,
|
||||||
|
|
|
@ -318,6 +318,7 @@ async fn venv_impl(
|
||||||
&flat_index,
|
&flat_index,
|
||||||
&state.index,
|
&state.index,
|
||||||
&state.git,
|
&state.git,
|
||||||
|
&state.capabilities,
|
||||||
&state.in_flight,
|
&state.in_flight,
|
||||||
index_strategy,
|
index_strategy,
|
||||||
&config_settings,
|
&config_settings,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue