mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 10:58:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Add a proxy layer for extras (#3100)

Browse source 
Given requirements like:

```
black==23.1.0
black[colorama]
```

The resolver will (on `main`) add a dependency on Black, and then try to
use the most recent version of Black to satisfy `black[colorama]`. For
sake of example, assume `black==24.0.0` is the most recent version. Once
the selects this most recent version, it'll fetch the metadata, then
return the dependencies for `black==24.0.0` with the `colorama` extra
enabled. Finally, it will tack on `black==24.0.0` (a dependency on the
base package). The resolver will then detect a conflict between
`black==23.1.0` and `black==24.0.0`, and throw out
`black[colorama]==24.0.0`, trying to next most-recent version.

This is both wasteful and can cause problems, since we're fetching
metadata for versions that will _never_ satisfy the resolver. In the
`apache-airflow[all]` case, I also ran into an issue whereby we were
attempting to build very old versions of `apache-airflow` due to
`apache-airflow[pandas]`, which in turn led to resolution failures.

The solution proposed here is that we create a new proxy package with
exactly two dependencies: one on `black` and one of `black[colorama]`.
Both of these packages must be at the same version as the proxy package,
so the resolver knows much _earlier_ that (in the above example) the
extra variant _must_ match `23.1.0`.
This commit is contained in:
Charlie Marsh 2024-04-18 21:04:59 -04:00 • committed by GitHub
parent 822ae19879
commit 2e88bb6f1b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 128 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

98
crates/uv-resolver/src/resolver/mod.rs
View file

@ -525,6 +525,7 @@ impl<
match package {
PubGrubPackage::Root(_) => {}
PubGrubPackage::Python(_) => {}
PubGrubPackage::Extra(_, _, _) => {}
PubGrubPackage::Package(name, _extra, None) => {
// Verify that the package is allowed under the hash-checking policy.
if !self.hasher.allows_package(name) {
@ -561,7 +562,7 @@ impl<
// Iterate over the potential packages, and fetch file metadata for any of them. These
// represent our current best guesses for the versions that we _might_ select.
for (package, range) in packages {
let PubGrubPackage::Package(package_name, _extra, None) = package else {
let PubGrubPackage::Package(package_name, None, None) = package else {
continue;
};
request_sink
@ -604,16 +605,9 @@ impl<
}
}
PubGrubPackage::Package(package_name, extra, Some(url)) => {
if let Some(extra) = extra {
debug!(
"Searching for a compatible version of {package_name}[{extra}] @ {url} ({range})",
);
} else {
debug!(
"Searching for a compatible version of {package_name} @ {url} ({range})"
);
}
PubGrubPackage::Extra(package_name, _, Some(url))
| PubGrubPackage::Package(package_name, _, Some(url)) => {
debug!("Searching for a compatible version of {package} @ {url} ({range})");
// If the dist is an editable, return the version from the editable metadata.
if let Some((_local, metadata)) = self.editables.get(package_name) {
@ -702,7 +696,8 @@ impl<
Ok(Some(ResolverVersion::Available(version.clone())))
}
PubGrubPackage::Package(package_name, extra, None) => {
PubGrubPackage::Extra(package_name, _, None)
| PubGrubPackage::Package(package_name, _, None) => {
// Wait for the metadata to be available.
let versions_response = self
.index
@ -732,13 +727,7 @@ impl<
}
};
if let Some(extra) = extra {
debug!(
"Searching for a compatible version of {package_name}[{extra}] ({range})",
);
} else {
debug!("Searching for a compatible version of {package_name} ({range})");
}
debug!("Searching for a compatible version of {package} ({range})");
// Find a version.
let Some(candidate) = self.selector.select(
@ -770,22 +759,13 @@ impl<
}
ResolvedDistRef::Installed(_) => Cow::Borrowed("installed"),
};
if let Some(extra) = extra {
debug!(
"Selecting: {}[{}]=={} ({})",
candidate.name(),
extra,
candidate.version(),
filename,
);
} else {
debug!(
"Selecting: {}=={} ({})",
candidate.name(),
candidate.version(),
filename,
);
}
debug!(
"Selecting: {}=={} ({})",
package,
candidate.version(),
filename,
);
// We want to return a package pinned to a specific version; but we _also_ want to
// store the exact file that we selected to satisfy that version.
@ -794,13 +774,16 @@ impl<
let version = candidate.version().clone();
// Emit a request to fetch the metadata for this version.
if self.index.distributions.register(candidate.version_id()) {
let request = match dist.for_resolution() {
ResolvedDistRef::Installable(dist) => Request::Dist(dist.clone()),
ResolvedDistRef::Installed(dist) => Request::Installed(dist.clone()),
};
request_sink.send(request).await?;
if matches!(package, PubGrubPackage::Package(_, _, _)) {
if self.index.distributions.register(candidate.version_id()) {
let request = match dist.for_resolution() {
ResolvedDistRef::Installable(dist) => Request::Dist(dist.clone()),
ResolvedDistRef::Installed(dist) => Request::Installed(dist.clone()),
};
request_sink.send(request).await?;
}
}
Ok(Some(ResolverVersion::Available(version)))
}
}
@ -896,7 +879,7 @@ impl<
// Determine if the distribution is editable.
if let Some((_local, metadata)) = self.editables.get(package_name) {
let mut constraints = PubGrubDependencies::from_requirements(
let constraints = PubGrubDependencies::from_requirements(
&metadata.requires_dist,
&self.constraints,
&self.overrides,
@ -917,14 +900,6 @@ impl<
self.visit_package(dep_package, request_sink).await?;
}
// If a package has an extra, insert a constraint on the base package.
if extra.is_some() {
constraints.push(
PubGrubPackage::Package(package_name.clone(), None, url.clone()),
Range::singleton(version.clone()),
);
}
return Ok(Dependencies::Available(constraints.into()));
}
@ -1013,7 +988,7 @@ impl<
}
};
let mut constraints = PubGrubDependencies::from_requirements(
let constraints = PubGrubDependencies::from_requirements(
&metadata.requires_dist,
&self.constraints,
&self.overrides,
@ -1034,16 +1009,20 @@ impl<
self.visit_package(package, request_sink).await?;
}
// If a package has an extra, insert a constraint on the base package.
if extra.is_some() {
constraints.push(
PubGrubPackage::Package(package_name.clone(), None, url.clone()),
Range::singleton(version.clone()),
);
}
Ok(Dependencies::Available(constraints.into()))
}
// Add a dependency on both the extra and base package.
PubGrubPackage::Extra(package_name, extra, url) => Ok(Dependencies::Available(vec![
(
PubGrubPackage::Package(package_name.clone(), None, url.clone()),
Range::singleton(version.clone()),
),
(
PubGrubPackage::Package(package_name.clone(), Some(extra.clone()), url.clone()),
Range::singleton(version.clone()),
),
])),
}
}
@ -1251,6 +1230,7 @@ impl<
match package {
PubGrubPackage::Root(_) => {}
PubGrubPackage::Python(_) => {}
PubGrubPackage::Extra(_, _, _) => {}
PubGrubPackage::Package(package_name, _extra, Some(url)) => {
reporter.on_progress(package_name, &VersionOrUrl::Url(url));
}