mirror of
https://github.com/astral-sh/uv.git
synced 2025-07-07 21:35:00 +00:00

Pubgrub stores incompatibilities as (package name, version range) tuples, meaning it needs to clone the package name for each incompatibility, and each non-borrowed operation on incompatibilities. https://github.com/astral-sh/uv/pull/3673 made me realize that `PubGrubPackage` has gotten large (expensive to copy), so like `Version` and other structs, i've added an `Arc` wrapper around it. It's a pity clippy forbids `.deref()`, it's less opaque than `&**` and has IDE support (clicking on `.deref()` jumps to the right impl). ## Benchmarks It looks like this matters most for complex resolutions which, i assume because they carry larger `PubGrubPackageInner::Package` and `PubGrubPackageInner::Extra` types. ```bash hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/jupyter.in" "./uv-branch pip compile -q ./scripts/requirements/jupyter.in" hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/airflow.in" "./uv-branch pip compile -q ./scripts/requirements/airflow.in" hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/boto3.in" "./uv-branch pip compile -q ./scripts/requirements/boto3.in" ``` ``` Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/jupyter.in Time (mean ± σ): 18.2 ms ± 1.6 ms [User: 14.4 ms, System: 26.0 ms] Range (min … max): 15.8 ms … 22.5 ms 181 runs Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/jupyter.in Time (mean ± σ): 17.8 ms ± 1.4 ms [User: 14.4 ms, System: 25.3 ms] Range (min … max): 15.4 ms … 23.1 ms 159 runs Summary ./uv-branch pip compile -q ./scripts/requirements/jupyter.in ran 1.02 ± 0.12 times faster than ./uv-main pip compile -q ./scripts/requirements/jupyter.in ``` ``` Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/airflow.in Time (mean ± σ): 153.7 ms ± 3.5 ms [User: 165.2 ms, System: 157.6 ms] Range (min … max): 150.4 ms … 163.0 ms 19 runs Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/airflow.in Time (mean ± σ): 123.9 ms ± 4.6 ms [User: 152.4 ms, System: 133.8 ms] Range (min … max): 118.4 ms … 138.1 ms 24 runs Summary ./uv-branch pip compile -q ./scripts/requirements/airflow.in ran 1.24 ± 0.05 times faster than ./uv-main pip compile -q ./scripts/requirements/airflow.in ``` ``` Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/boto3.in Time (mean ± σ): 327.0 ms ± 3.8 ms [User: 344.5 ms, System: 71.6 ms] Range (min … max): 322.7 ms … 334.6 ms 10 runs Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/boto3.in Time (mean ± σ): 311.2 ms ± 3.1 ms [User: 339.3 ms, System: 63.1 ms] Range (min … max): 307.8 ms … 317.0 ms 10 runs Summary ./uv-branch pip compile -q ./scripts/requirements/boto3.in ran 1.05 ± 0.02 times faster than ./uv-main pip compile -q ./scripts/requirements/boto3.in ``` <!-- Thank you for contributing to uv! To help us out with reviewing, please consider the following: - Does this pull request include a summary of the change? (See below.) - Does this pull request include a descriptive title? - Does this pull request include references to any relevant issues? -->
343 lines
13 KiB
Rust
343 lines
13 KiB
Rust
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::fmt::Formatter;
|
|
use std::ops::Deref;
|
|
use std::sync::Arc;
|
|
|
|
use indexmap::IndexMap;
|
|
use pubgrub::range::Range;
|
|
use pubgrub::report::{DefaultStringReporter, DerivationTree, External, Reporter};
|
|
use rustc_hash::{FxHashMap, FxHashSet};
|
|
|
|
use dashmap::DashMap;
|
|
use distribution_types::{BuiltDist, IndexLocations, InstalledDist, ParsedUrlError, SourceDist};
|
|
use pep440_rs::Version;
|
|
use pep508_rs::Requirement;
|
|
use uv_normalize::PackageName;
|
|
|
|
use crate::candidate_selector::CandidateSelector;
|
|
use crate::dependency_provider::UvDependencyProvider;
|
|
use crate::pubgrub::{PubGrubPackage, PubGrubPackageInner, PubGrubPython, PubGrubReportFormatter};
|
|
use crate::python_requirement::PythonRequirement;
|
|
use crate::resolver::{
|
|
FxOnceMap, IncompletePackage, UnavailablePackage, UnavailableReason, VersionsResponse,
|
|
};
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum ResolveError {
|
|
#[error("Failed to find a version of `{0}` that satisfies the requirement")]
|
|
NotFound(Requirement),
|
|
|
|
#[error(transparent)]
|
|
Client(#[from] uv_client::Error),
|
|
|
|
#[error("The channel closed unexpectedly")]
|
|
ChannelClosed,
|
|
|
|
#[error(transparent)]
|
|
Join(#[from] tokio::task::JoinError),
|
|
|
|
#[error("Attempted to wait on an unregistered task")]
|
|
Unregistered,
|
|
|
|
#[error("Package metadata name `{metadata}` does not match given name `{given}`")]
|
|
NameMismatch {
|
|
given: PackageName,
|
|
metadata: PackageName,
|
|
},
|
|
|
|
#[error("~= operator requires at least two release segments: `{0}`")]
|
|
InvalidTildeEquals(pep440_rs::VersionSpecifier),
|
|
|
|
#[error("Requirements contain conflicting URLs for package `{0}`:\n- {1}\n- {2}")]
|
|
ConflictingUrlsDirect(PackageName, String, String),
|
|
|
|
#[error("There are conflicting URLs for package `{0}`:\n- {1}\n- {2}")]
|
|
ConflictingUrlsTransitive(PackageName, String, String),
|
|
|
|
#[error("Package `{0}` attempted to resolve via URL: {1}. URL dependencies must be expressed as direct requirements or constraints. Consider adding `{0} @ {1}` to your dependencies or constraints file.")]
|
|
DisallowedUrl(PackageName, String),
|
|
|
|
#[error("There are conflicting editable requirements for package `{0}`:\n- {1}\n- {2}")]
|
|
ConflictingEditables(PackageName, String, String),
|
|
|
|
#[error(transparent)]
|
|
DistributionType(#[from] distribution_types::Error),
|
|
|
|
#[error("Failed to download `{0}`")]
|
|
Fetch(Box<BuiltDist>, #[source] uv_distribution::Error),
|
|
|
|
#[error("Failed to download and build `{0}`")]
|
|
FetchAndBuild(Box<SourceDist>, #[source] uv_distribution::Error),
|
|
|
|
#[error("Failed to read `{0}`")]
|
|
Read(Box<BuiltDist>, #[source] uv_distribution::Error),
|
|
|
|
// TODO(zanieb): Use `thiserror` in `InstalledDist` so we can avoid chaining `anyhow`
|
|
#[error("Failed to read metadata from installed package `{0}`")]
|
|
ReadInstalled(Box<InstalledDist>, #[source] anyhow::Error),
|
|
|
|
#[error("Failed to build `{0}`")]
|
|
Build(Box<SourceDist>, #[source] uv_distribution::Error),
|
|
|
|
#[error(transparent)]
|
|
NoSolution(#[from] NoSolutionError),
|
|
|
|
#[error("{package} {version} depends on itself")]
|
|
SelfDependency {
|
|
/// Package whose dependencies we want.
|
|
package: Box<PubGrubPackage>,
|
|
/// Version of the package for which we want the dependencies.
|
|
version: Box<Version>,
|
|
},
|
|
|
|
#[error("Attempted to construct an invalid version specifier")]
|
|
InvalidVersion(#[from] pep440_rs::VersionSpecifierBuildError),
|
|
|
|
#[error("In `--require-hashes` mode, all requirements must be pinned upfront with `==`, but found: `{0}`")]
|
|
UnhashedPackage(PackageName),
|
|
|
|
// TODO(konsti): Attach the distribution that contained the invalid requirement as error source.
|
|
#[error("Failed to parse requirements")]
|
|
DirectUrl(#[from] Box<ParsedUrlError>),
|
|
|
|
/// Something unexpected happened.
|
|
#[error("{0}")]
|
|
Failure(String),
|
|
}
|
|
|
|
impl<T> From<tokio::sync::mpsc::error::SendError<T>> for ResolveError {
|
|
/// Drop the value we want to send to not leak the private type we're sending.
|
|
/// The tokio error only says "channel closed", so we don't lose information.
|
|
fn from(_value: tokio::sync::mpsc::error::SendError<T>) -> Self {
|
|
Self::ChannelClosed
|
|
}
|
|
}
|
|
|
|
/// Given a [`DerivationTree`], collapse any [`External::FromDependencyOf`] incompatibilities
|
|
/// wrap an [`PubGrubPackageInner::Extra`] package.
|
|
fn collapse_extra_proxies(
|
|
derivation_tree: &mut DerivationTree<PubGrubPackage, Range<Version>, UnavailableReason>,
|
|
) {
|
|
match derivation_tree {
|
|
DerivationTree::External(_) => {}
|
|
DerivationTree::Derived(derived) => {
|
|
match (
|
|
Arc::make_mut(&mut derived.cause1),
|
|
Arc::make_mut(&mut derived.cause2),
|
|
) {
|
|
(
|
|
DerivationTree::External(External::FromDependencyOf(package, ..)),
|
|
ref mut cause,
|
|
) if matches!(&**package, PubGrubPackageInner::Extra { .. }) => {
|
|
collapse_extra_proxies(cause);
|
|
*derivation_tree = cause.clone();
|
|
}
|
|
(
|
|
ref mut cause,
|
|
DerivationTree::External(External::FromDependencyOf(package, ..)),
|
|
) if matches!(&**package, PubGrubPackageInner::Extra { .. }) => {
|
|
collapse_extra_proxies(cause);
|
|
*derivation_tree = cause.clone();
|
|
}
|
|
_ => {
|
|
collapse_extra_proxies(Arc::make_mut(&mut derived.cause1));
|
|
collapse_extra_proxies(Arc::make_mut(&mut derived.cause2));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<pubgrub::error::PubGrubError<UvDependencyProvider>> for ResolveError {
|
|
fn from(value: pubgrub::error::PubGrubError<UvDependencyProvider>) -> Self {
|
|
match value {
|
|
// These are all never type variant that can never match, but never is experimental
|
|
pubgrub::error::PubGrubError::ErrorChoosingPackageVersion(_)
|
|
| pubgrub::error::PubGrubError::ErrorInShouldCancel(_)
|
|
| pubgrub::error::PubGrubError::ErrorRetrievingDependencies { .. } => {
|
|
unreachable!()
|
|
}
|
|
pubgrub::error::PubGrubError::Failure(inner) => Self::Failure(inner),
|
|
pubgrub::error::PubGrubError::NoSolution(mut derivation_tree) => {
|
|
collapse_extra_proxies(&mut derivation_tree);
|
|
|
|
Self::NoSolution(NoSolutionError {
|
|
derivation_tree,
|
|
// The following should be populated before display for the best error messages
|
|
available_versions: IndexMap::default(),
|
|
selector: None,
|
|
python_requirement: None,
|
|
index_locations: None,
|
|
unavailable_packages: FxHashMap::default(),
|
|
incomplete_packages: FxHashMap::default(),
|
|
})
|
|
}
|
|
pubgrub::error::PubGrubError::SelfDependency { package, version } => {
|
|
Self::SelfDependency {
|
|
package: Box::new(package),
|
|
version: Box::new(version),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A wrapper around [`pubgrub::error::PubGrubError::NoSolution`] that displays a resolution failure report.
|
|
#[derive(Debug)]
|
|
pub struct NoSolutionError {
|
|
derivation_tree: DerivationTree<PubGrubPackage, Range<Version>, UnavailableReason>,
|
|
available_versions: IndexMap<PubGrubPackage, BTreeSet<Version>>,
|
|
selector: Option<CandidateSelector>,
|
|
python_requirement: Option<PythonRequirement>,
|
|
index_locations: Option<IndexLocations>,
|
|
unavailable_packages: FxHashMap<PackageName, UnavailablePackage>,
|
|
incomplete_packages: FxHashMap<PackageName, BTreeMap<Version, IncompletePackage>>,
|
|
}
|
|
|
|
impl std::error::Error for NoSolutionError {}
|
|
|
|
impl std::fmt::Display for NoSolutionError {
|
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
// Write the derivation report.
|
|
let formatter = PubGrubReportFormatter {
|
|
available_versions: &self.available_versions,
|
|
python_requirement: self.python_requirement.as_ref(),
|
|
};
|
|
let report =
|
|
DefaultStringReporter::report_with_formatter(&self.derivation_tree, &formatter);
|
|
write!(f, "{report}")?;
|
|
|
|
// Include any additional hints.
|
|
for hint in formatter.hints(
|
|
&self.derivation_tree,
|
|
&self.selector,
|
|
&self.index_locations,
|
|
&self.unavailable_packages,
|
|
&self.incomplete_packages,
|
|
) {
|
|
write!(f, "\n\n{hint}")?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl NoSolutionError {
|
|
/// Update the available versions attached to the error using the given package version index.
|
|
///
|
|
/// Only packages used in the error's derivation tree will be retrieved.
|
|
#[must_use]
|
|
pub(crate) fn with_available_versions(
|
|
mut self,
|
|
python_requirement: &PythonRequirement,
|
|
visited: &FxHashSet<PackageName>,
|
|
package_versions: &FxOnceMap<PackageName, Arc<VersionsResponse>>,
|
|
) -> Self {
|
|
let mut available_versions = IndexMap::default();
|
|
for package in self.derivation_tree.packages() {
|
|
match &**package {
|
|
PubGrubPackageInner::Root(_) => {}
|
|
PubGrubPackageInner::Python(PubGrubPython::Installed) => {
|
|
available_versions.insert(
|
|
package.clone(),
|
|
BTreeSet::from([python_requirement.installed().deref().clone()]),
|
|
);
|
|
}
|
|
PubGrubPackageInner::Python(PubGrubPython::Target) => {
|
|
available_versions.insert(
|
|
package.clone(),
|
|
BTreeSet::from([python_requirement.target().deref().clone()]),
|
|
);
|
|
}
|
|
PubGrubPackageInner::Extra { .. } => {}
|
|
PubGrubPackageInner::Package { name, .. } => {
|
|
// Avoid including available versions for packages that exist in the derivation
|
|
// tree, but were never visited during resolution. We _may_ have metadata for
|
|
// these packages, but it's non-deterministic, and omitting them ensures that
|
|
// we represent the state of the resolver at the time of failure.
|
|
if visited.contains(name) {
|
|
if let Some(response) = package_versions.get(name) {
|
|
if let VersionsResponse::Found(ref version_maps) = *response {
|
|
for version_map in version_maps {
|
|
available_versions
|
|
.entry(package.clone())
|
|
.or_insert_with(BTreeSet::new)
|
|
.extend(
|
|
version_map.iter().map(|(version, _)| version.clone()),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.available_versions = available_versions;
|
|
self
|
|
}
|
|
|
|
/// Update the candidate selector attached to the error.
|
|
#[must_use]
|
|
pub(crate) fn with_selector(mut self, selector: CandidateSelector) -> Self {
|
|
self.selector = Some(selector);
|
|
self
|
|
}
|
|
|
|
/// Update the index locations attached to the error.
|
|
#[must_use]
|
|
pub(crate) fn with_index_locations(mut self, index_locations: &IndexLocations) -> Self {
|
|
self.index_locations = Some(index_locations.clone());
|
|
self
|
|
}
|
|
|
|
/// Update the unavailable packages attached to the error.
|
|
#[must_use]
|
|
pub(crate) fn with_unavailable_packages(
|
|
mut self,
|
|
unavailable_packages: &DashMap<PackageName, UnavailablePackage>,
|
|
) -> Self {
|
|
let mut new = FxHashMap::default();
|
|
for package in self.derivation_tree.packages() {
|
|
if let PubGrubPackageInner::Package { name, .. } = &**package {
|
|
if let Some(reason) = unavailable_packages.get(name) {
|
|
new.insert(name.clone(), reason.clone());
|
|
}
|
|
}
|
|
}
|
|
self.unavailable_packages = new;
|
|
self
|
|
}
|
|
|
|
/// Update the incomplete packages attached to the error.
|
|
#[must_use]
|
|
pub(crate) fn with_incomplete_packages(
|
|
mut self,
|
|
incomplete_packages: &DashMap<PackageName, DashMap<Version, IncompletePackage>>,
|
|
) -> Self {
|
|
let mut new = FxHashMap::default();
|
|
for package in self.derivation_tree.packages() {
|
|
if let PubGrubPackageInner::Package { name, .. } = &**package {
|
|
if let Some(versions) = incomplete_packages.get(name) {
|
|
for entry in versions.iter() {
|
|
let (version, reason) = entry.pair();
|
|
new.entry(name.clone())
|
|
.or_insert_with(BTreeMap::default)
|
|
.insert(version.clone(), reason.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.incomplete_packages = new;
|
|
self
|
|
}
|
|
|
|
/// Update the Python requirements attached to the error.
|
|
#[must_use]
|
|
pub(crate) fn with_python_requirement(
|
|
mut self,
|
|
python_requirement: &PythonRequirement,
|
|
) -> Self {
|
|
self.python_requirement = Some(python_requirement.clone());
|
|
self
|
|
}
|
|
}
|