mirror of
https://github.com/astral-sh/uv.git
synced 2025-08-04 02:48:17 +00:00
Add support for named and explicit indexes (#7481)
## Summary This PR adds a first-class API for defining registry indexes, beyond our existing `--index-url` and `--extra-index-url` setup. Specifically, you now define indexes like so in a `uv.toml` or `pyproject.toml` file: ```toml [[tool.uv.index]] name = "pytorch" url = "https://download.pytorch.org/whl/cu121" ``` You can also provide indexes via `--index` and `UV_INDEX`, and override the default index with `--default-index` and `UV_DEFAULT_INDEX`. ### Index priority Indexes are prioritized in the order in which they're defined, such that the first-defined index has highest priority. Indexes are also inherited from parent configuration (e.g., the user-level `uv.toml`), but are placed after any indexes in the current project, matching our semantics for other array-based configuration values. You can mix `--index` and `--default-index` with the legacy `--index-url` and `--extra-index-url` settings; the latter two are merely treated as unnamed `[[tool.uv.index]]` entries. ### Index pinning If an index includes a name (which is optional), it can then be referenced via `tool.uv.sources`: ```toml [[tool.uv.index]] name = "pytorch" url = "https://download.pytorch.org/whl/cu121" [tool.uv.sources] torch = { index = "pytorch" } ``` If an index is marked as `explicit = true`, it can _only_ be used via such references, and will never be searched implicitly: ```toml [[tool.uv.index]] name = "pytorch" url = "https://download.pytorch.org/whl/cu121" explicit = true [tool.uv.sources] torch = { index = "pytorch" } ``` Indexes defined outside of the current project (e.g., in the user-level `uv.toml`) can _not_ be explicitly selected. (As of now, we only support using a single index for a given `tool.uv.sources` definition.) ### Default index By default, we include PyPI as the default index. This remains true even if the user defines a `[[tool.uv.index]]` -- PyPI is still used as a fallback. You can mark an index as `default = true` to (1) disable the use of PyPI, and (2) bump it to the bottom of the prioritized list, such that it's used only if a package does not exist on a prior index: ```toml [[tool.uv.index]] name = "pytorch" url = "https://download.pytorch.org/whl/cu121" default = true ``` ### Name reuse If a name is reused, the higher-priority index with that name is used, while the lower-priority indexes are ignored entirely. For example, given: ```toml [[tool.uv.index]] name = "pytorch" url = "https://download.pytorch.org/whl/cu121" [[tool.uv.index]] name = "pytorch" url = "https://test.pypi.org/simple" ``` The `https://test.pypi.org/simple` index would be ignored entirely, since it's lower-priority than `https://download.pytorch.org/whl/cu121` but shares the same name. Closes #171. ## Future work - Users should be able to provide authentication for named indexes via environment variables. - `uv add` should automatically write `--index` entries to the `pyproject.toml` file. - Users should be able to provide multiple indexes for a given package, stratified by platform: ```toml [tool.uv.sources] torch = [ { index = "cpu", markers = "sys_platform == 'darwin'" }, { index = "gpu", markers = "sys_platform != 'darwin'" }, ] ``` - Users should be able to specify a proxy URL for a given index, to avoid writing user-specific URLs to a lockfile: ```toml [[tool.uv.index]] name = "test" url = "https://private.org/simple" proxy = "http://<omitted>/pypi/simple" ```
This commit is contained in:
Charlie Marsh
GitHub
parent
34be3af84f
commit
5b391770df
51 changed files with 3526 additions and 658 deletions
|
|
@ -53,6 +53,9 @@ pub enum ResolveError {
|
|||
fork_markers: MarkerTree,
|
||||
},
|
||||
|
||||
#[error("Requirements contain conflicting indexes for package `{0}`: `{1}` vs. `{2}`")]
|
||||
ConflictingIndexes(PackageName, String, String),
|
||||
|
||||
#[error("Package `{0}` attempted to resolve via URL: {1}. URL dependencies must be expressed as direct requirements or constraints. Consider adding `{0} @ {1}` to your dependencies or constraints file.")]
|
||||
DisallowedUrl(PackageName, String),
|
||||
|
||||
|
|
|
|||
|
|
@ -1058,10 +1058,10 @@ impl Lock {
|
|||
// Collect the set of available indexes (both `--index-url` and `--find-links` entries).
|
||||
let remotes = indexes.map(|locations| {
|
||||
locations
|
||||
.indexes()
|
||||
.filter_map(|index_url| match index_url {
|
||||
.allowed_indexes()
|
||||
.filter_map(|index| match index.url() {
|
||||
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
|
||||
Some(UrlString::from(index_url.redacted()))
|
||||
Some(UrlString::from(index.url().redacted()))
|
||||
}
|
||||
IndexUrl::Path(_) => None,
|
||||
})
|
||||
|
|
@ -1080,11 +1080,11 @@ impl Lock {
|
|||
|
||||
let locals = indexes.map(|locations| {
|
||||
locations
|
||||
.indexes()
|
||||
.filter_map(|index_url| match index_url {
|
||||
.allowed_indexes()
|
||||
.filter_map(|index| match index.url() {
|
||||
IndexUrl::Pypi(_) | IndexUrl::Url(_) => None,
|
||||
IndexUrl::Path(index_url) => {
|
||||
let path = index_url.to_file_path().ok()?;
|
||||
IndexUrl::Path(url) => {
|
||||
let path = url.to_file_path().ok()?;
|
||||
let path = relative_to(&path, workspace.install_path())
|
||||
.or_else(|_| std::path::absolute(path))
|
||||
.ok()?;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use pubgrub::{DerivationTree, Derived, External, Map, Range, ReportFormatter, Te
|
|||
use rustc_hash::FxHashMap;
|
||||
|
||||
use uv_configuration::IndexStrategy;
|
||||
use uv_distribution_types::{IndexLocations, IndexUrl};
|
||||
use uv_distribution_types::{Index, IndexLocations, IndexUrl};
|
||||
use uv_normalize::PackageName;
|
||||
use uv_pep440::Version;
|
||||
|
||||
|
|
@ -708,6 +708,7 @@ impl PubGrubReportFormatter<'_> {
|
|||
// indexes were not queried, and could contain a compatible version.
|
||||
if let Some(next_index) = index_locations
|
||||
.indexes()
|
||||
.map(Index::url)
|
||||
.skip_while(|url| *url != found_index)
|
||||
.nth(1)
|
||||
{
|
||||
|
|
|
|||
67
crates/uv-resolver/src/resolver/indexes.rs
Normal file
67
crates/uv-resolver/src/resolver/indexes.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
use crate::{DependencyMode, Manifest, ResolveError, ResolverMarkers};
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
use std::collections::hash_map::Entry;
|
||||
use uv_distribution_types::IndexUrl;
|
||||
use uv_normalize::PackageName;
|
||||
use uv_pep508::VerbatimUrl;
|
||||
use uv_pypi_types::RequirementSource;
|
||||
|
||||
/// A map of package names to their explicit index.
|
||||
///
|
||||
/// For example, given:
|
||||
/// ```toml
|
||||
/// [[tool.uv.index]]
|
||||
/// name = "pytorch"
|
||||
/// url = "https://download.pytorch.org/whl/cu121"
|
||||
///
|
||||
/// [tool.uv.sources]
|
||||
/// torch = { index = "pytorch" }
|
||||
/// ```
|
||||
///
|
||||
/// [`Indexes`] would contain a single entry mapping `torch` to `https://download.pytorch.org/whl/cu121`.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(crate) struct Indexes(FxHashMap<PackageName, IndexUrl>);
|
||||
|
||||
impl Indexes {
|
||||
/// Determine the set of explicit, pinned indexes in the [`Manifest`].
|
||||
pub(crate) fn from_manifest(
|
||||
manifest: &Manifest,
|
||||
markers: &ResolverMarkers,
|
||||
dependencies: DependencyMode,
|
||||
) -> Result<Self, ResolveError> {
|
||||
let mut indexes = FxHashMap::<PackageName, IndexUrl>::default();
|
||||
|
||||
for requirement in manifest.requirements(markers, dependencies) {
|
||||
let RequirementSource::Registry {
|
||||
index: Some(index), ..
|
||||
} = &requirement.source
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let index = IndexUrl::from(VerbatimUrl::from_url(index.clone()));
|
||||
match indexes.entry(requirement.name.clone()) {
|
||||
Entry::Occupied(entry) => {
|
||||
let existing = entry.get();
|
||||
if *existing != index {
|
||||
return Err(ResolveError::ConflictingIndexes(
|
||||
requirement.name.clone(),
|
||||
existing.to_string(),
|
||||
index.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self(indexes))
|
||||
}
|
||||
|
||||
/// Return the explicit index for a given [`PackageName`].
|
||||
pub(crate) fn get(&self, package_name: &PackageName) -> Option<&IndexUrl> {
|
||||
self.0.get(package_name)
|
||||
}
|
||||
}
|
||||
|
|
@ -28,8 +28,9 @@ use uv_configuration::{Constraints, Overrides};
|
|||
use uv_distribution::{ArchiveMetadata, DistributionDatabase};
|
||||
use uv_distribution_types::{
|
||||
BuiltDist, CompatibleDist, Dist, DistributionMetadata, IncompatibleDist, IncompatibleSource,
|
||||
IncompatibleWheel, IndexCapabilities, IndexLocations, InstalledDist, PythonRequirementKind,
|
||||
RemoteSource, ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef,
|
||||
IncompatibleWheel, IndexCapabilities, IndexLocations, IndexUrl, InstalledDist,
|
||||
PythonRequirementKind, RemoteSource, ResolvedDist, ResolvedDistRef, SourceDist,
|
||||
VersionOrUrlRef,
|
||||
};
|
||||
use uv_git::GitResolver;
|
||||
use uv_normalize::{ExtraName, GroupName, PackageName};
|
||||
|
|
@ -60,6 +61,7 @@ pub(crate) use crate::resolver::availability::{
|
|||
use crate::resolver::batch_prefetch::BatchPrefetcher;
|
||||
use crate::resolver::groups::Groups;
|
||||
pub use crate::resolver::index::InMemoryIndex;
|
||||
use crate::resolver::indexes::Indexes;
|
||||
pub use crate::resolver::provider::{
|
||||
DefaultResolverProvider, MetadataResponse, PackageVersionsResult, ResolverProvider,
|
||||
VersionsResponse, WheelMetadataResult,
|
||||
|
|
@ -74,6 +76,7 @@ mod batch_prefetch;
|
|||
mod fork_map;
|
||||
mod groups;
|
||||
mod index;
|
||||
mod indexes;
|
||||
mod locals;
|
||||
mod provider;
|
||||
mod reporter;
|
||||
|
|
@ -100,6 +103,7 @@ struct ResolverState<InstalledPackages: InstalledPackagesProvider> {
|
|||
exclusions: Exclusions,
|
||||
urls: Urls,
|
||||
locals: Locals,
|
||||
indexes: Indexes,
|
||||
dependency_mode: DependencyMode,
|
||||
hasher: HashStrategy,
|
||||
markers: ResolverMarkers,
|
||||
|
|
@ -204,6 +208,7 @@ impl<Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvider>
|
|||
dependency_mode: options.dependency_mode,
|
||||
urls: Urls::from_manifest(&manifest, &markers, git, options.dependency_mode)?,
|
||||
locals: Locals::from_manifest(&manifest, &markers, options.dependency_mode),
|
||||
indexes: Indexes::from_manifest(&manifest, &markers, options.dependency_mode)?,
|
||||
groups: Groups::from_manifest(&manifest, &markers),
|
||||
project: manifest.project,
|
||||
workspace_members: manifest.workspace_members,
|
||||
|
|
@ -377,7 +382,9 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
continue 'FORK;
|
||||
};
|
||||
state.next = highest_priority_pkg;
|
||||
|
||||
let url = state.next.name().and_then(|name| state.fork_urls.get(name));
|
||||
let index = state.next.name().and_then(|name| self.indexes.get(name));
|
||||
|
||||
// Consider:
|
||||
// ```toml
|
||||
|
|
@ -390,7 +397,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
// since we weren't sure whether it might also be a URL requirement when
|
||||
// transforming the requirements. For that case, we do another request here
|
||||
// (idempotent due to caching).
|
||||
self.request_package(&state.next, url, &request_sink)?;
|
||||
self.request_package(&state.next, url, index, &request_sink)?;
|
||||
|
||||
prefetcher.version_tried(state.next.clone());
|
||||
|
||||
|
|
@ -529,7 +536,8 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
url: _,
|
||||
} = dependency;
|
||||
let url = package.name().and_then(|name| state.fork_urls.get(name));
|
||||
self.visit_package(package, url, &request_sink)?;
|
||||
let index = package.name().and_then(|name| self.indexes.get(name));
|
||||
self.visit_package(package, url, index, &request_sink)?;
|
||||
}
|
||||
}
|
||||
ForkedDependencies::Forked {
|
||||
|
|
@ -697,7 +705,8 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
let url = package
|
||||
.name()
|
||||
.and_then(|name| forked_state.fork_urls.get(name));
|
||||
self.visit_package(package, url, request_sink)?;
|
||||
let index = package.name().and_then(|name| self.indexes.get(name));
|
||||
self.visit_package(package, url, index, request_sink)?;
|
||||
}
|
||||
Ok(forked_state)
|
||||
})
|
||||
|
|
@ -720,6 +729,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
&self,
|
||||
package: &PubGrubPackage,
|
||||
url: Option<&VerbatimParsedUrl>,
|
||||
index: Option<&IndexUrl>,
|
||||
request_sink: &Sender<Request>,
|
||||
) -> Result<(), ResolveError> {
|
||||
// Ignore unresolved URL packages.
|
||||
|
|
@ -732,13 +742,14 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
self.request_package(package, url, request_sink)
|
||||
self.request_package(package, url, index, request_sink)
|
||||
}
|
||||
|
||||
fn request_package(
|
||||
&self,
|
||||
package: &PubGrubPackage,
|
||||
url: Option<&VerbatimParsedUrl>,
|
||||
index: Option<&IndexUrl>,
|
||||
request_sink: &Sender<Request>,
|
||||
) -> Result<(), ResolveError> {
|
||||
// Only request real package
|
||||
|
|
@ -760,7 +771,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
} else {
|
||||
// Emit a request to fetch the metadata for this package.
|
||||
if self.index.packages().register(name.clone()) {
|
||||
request_sink.blocking_send(Request::Package(name.clone()))?;
|
||||
request_sink.blocking_send(Request::Package(name.clone(), index.cloned()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -1709,9 +1720,9 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
|||
) -> Result<Option<Response>, ResolveError> {
|
||||
match request {
|
||||
// Fetch package metadata from the registry.
|
||||
Request::Package(package_name) => {
|
||||
Request::Package(package_name, index) => {
|
||||
let package_versions = provider
|
||||
.get_package_versions(&package_name)
|
||||
.get_package_versions(&package_name, index.as_ref())
|
||||
.boxed_local()
|
||||
.await
|
||||
.map_err(ResolveError::Client)?;
|
||||
|
|
@ -2505,7 +2516,7 @@ impl ResolutionPackage {
|
|||
#[allow(clippy::large_enum_variant)]
|
||||
pub(crate) enum Request {
|
||||
/// A request to fetch the metadata for a package.
|
||||
Package(PackageName),
|
||||
Package(PackageName, Option<IndexUrl>),
|
||||
/// A request to fetch the metadata for a built or source distribution.
|
||||
Dist(Dist),
|
||||
/// A request to fetch the metadata from an already-installed distribution.
|
||||
|
|
@ -2554,7 +2565,7 @@ impl<'a> From<ResolvedDistRef<'a>> for Request {
|
|||
impl Display for Request {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Package(package_name) => {
|
||||
Self::Package(package_name, _) => {
|
||||
write!(f, "Versions {package_name}")
|
||||
}
|
||||
Self::Dist(dist) => {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use std::future::Future;
|
|||
|
||||
use uv_configuration::BuildOptions;
|
||||
use uv_distribution::{ArchiveMetadata, DistributionDatabase};
|
||||
use uv_distribution_types::Dist;
|
||||
use uv_distribution_types::{Dist, IndexUrl};
|
||||
use uv_normalize::PackageName;
|
||||
use uv_platform_tags::Tags;
|
||||
use uv_types::{BuildContext, HashStrategy};
|
||||
|
|
@ -49,6 +49,7 @@ pub trait ResolverProvider {
|
|||
fn get_package_versions<'io>(
|
||||
&'io self,
|
||||
package_name: &'io PackageName,
|
||||
index: Option<&'io IndexUrl>,
|
||||
) -> impl Future<Output = PackageVersionsResult> + 'io;
|
||||
|
||||
/// Get the metadata for a distribution.
|
||||
|
|
@ -111,11 +112,12 @@ impl<'a, Context: BuildContext> ResolverProvider for DefaultResolverProvider<'a,
|
|||
async fn get_package_versions<'io>(
|
||||
&'io self,
|
||||
package_name: &'io PackageName,
|
||||
index: Option<&'io IndexUrl>,
|
||||
) -> PackageVersionsResult {
|
||||
let result = self
|
||||
.fetcher
|
||||
.client()
|
||||
.managed(|client| client.simple(package_name))
|
||||
.managed(|client| client.simple(package_name, index))
|
||||
.await;
|
||||
|
||||
match result {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue