uv/crates/uv-resolver/src/error.rs
konsti 76418f5bdf
Arc-wrap PubGrubPackage for cheap cloning in pubgrub (#3688)
Pubgrub stores incompatibilities as (package name, version range)
tuples, meaning it needs to clone the package name for each
incompatibility, and each non-borrowed operation on incompatibilities.
https://github.com/astral-sh/uv/pull/3673 made me realize that
`PubGrubPackage` has gotten large (expensive to copy), so like `Version`
and other structs, i've added an `Arc` wrapper around it.

It's a pity clippy forbids `.deref()`, it's less opaque than `&**` and
has IDE support (clicking on `.deref()` jumps to the right impl).

## Benchmarks

It looks like this matters most for complex resolutions which, i assume
because they carry larger `PubGrubPackageInner::Package` and
`PubGrubPackageInner::Extra` types.

```bash
hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/jupyter.in" "./uv-branch pip compile -q ./scripts/requirements/jupyter.in"
hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/airflow.in" "./uv-branch pip compile -q ./scripts/requirements/airflow.in"
hyperfine --warmup 5 "./uv-main pip compile -q ./scripts/requirements/boto3.in" "./uv-branch pip compile -q ./scripts/requirements/boto3.in"
```

```
Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/jupyter.in
  Time (mean ± σ):      18.2 ms ±   1.6 ms    [User: 14.4 ms, System: 26.0 ms]
  Range (min … max):    15.8 ms …  22.5 ms    181 runs

Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/jupyter.in
  Time (mean ± σ):      17.8 ms ±   1.4 ms    [User: 14.4 ms, System: 25.3 ms]
  Range (min … max):    15.4 ms …  23.1 ms    159 runs

Summary
  ./uv-branch pip compile -q ./scripts/requirements/jupyter.in ran
    1.02 ± 0.12 times faster than ./uv-main pip compile -q ./scripts/requirements/jupyter.in
```

```
Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/airflow.in
  Time (mean ± σ):     153.7 ms ±   3.5 ms    [User: 165.2 ms, System: 157.6 ms]
  Range (min … max):   150.4 ms … 163.0 ms    19 runs

Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/airflow.in
  Time (mean ± σ):     123.9 ms ±   4.6 ms    [User: 152.4 ms, System: 133.8 ms]
  Range (min … max):   118.4 ms … 138.1 ms    24 runs

Summary
  ./uv-branch pip compile -q ./scripts/requirements/airflow.in ran
    1.24 ± 0.05 times faster than ./uv-main pip compile -q ./scripts/requirements/airflow.in
```

```
Benchmark 1: ./uv-main pip compile -q ./scripts/requirements/boto3.in
  Time (mean ± σ):     327.0 ms ±   3.8 ms    [User: 344.5 ms, System: 71.6 ms]
  Range (min … max):   322.7 ms … 334.6 ms    10 runs

Benchmark 2: ./uv-branch pip compile -q ./scripts/requirements/boto3.in
  Time (mean ± σ):     311.2 ms ±   3.1 ms    [User: 339.3 ms, System: 63.1 ms]
  Range (min … max):   307.8 ms … 317.0 ms    10 runs

Summary
  ./uv-branch pip compile -q ./scripts/requirements/boto3.in ran
    1.05 ± 0.02 times faster than ./uv-main pip compile -q ./scripts/requirements/boto3.in
```

<!--
Thank you for contributing to uv! To help us out with reviewing, please
consider the following:

- Does this pull request include a summary of the change? (See below.)
- Does this pull request include a descriptive title?
- Does this pull request include references to any relevant issues?
-->
2024-05-21 13:49:35 +02:00

343 lines
13 KiB
Rust

use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Formatter;
use std::ops::Deref;
use std::sync::Arc;
use indexmap::IndexMap;
use pubgrub::range::Range;
use pubgrub::report::{DefaultStringReporter, DerivationTree, External, Reporter};
use rustc_hash::{FxHashMap, FxHashSet};
use dashmap::DashMap;
use distribution_types::{BuiltDist, IndexLocations, InstalledDist, ParsedUrlError, SourceDist};
use pep440_rs::Version;
use pep508_rs::Requirement;
use uv_normalize::PackageName;
use crate::candidate_selector::CandidateSelector;
use crate::dependency_provider::UvDependencyProvider;
use crate::pubgrub::{PubGrubPackage, PubGrubPackageInner, PubGrubPython, PubGrubReportFormatter};
use crate::python_requirement::PythonRequirement;
use crate::resolver::{
FxOnceMap, IncompletePackage, UnavailablePackage, UnavailableReason, VersionsResponse,
};
#[derive(Debug, thiserror::Error)]
pub enum ResolveError {
#[error("Failed to find a version of `{0}` that satisfies the requirement")]
NotFound(Requirement),
#[error(transparent)]
Client(#[from] uv_client::Error),
#[error("The channel closed unexpectedly")]
ChannelClosed,
#[error(transparent)]
Join(#[from] tokio::task::JoinError),
#[error("Attempted to wait on an unregistered task")]
Unregistered,
#[error("Package metadata name `{metadata}` does not match given name `{given}`")]
NameMismatch {
given: PackageName,
metadata: PackageName,
},
#[error("~= operator requires at least two release segments: `{0}`")]
InvalidTildeEquals(pep440_rs::VersionSpecifier),
#[error("Requirements contain conflicting URLs for package `{0}`:\n- {1}\n- {2}")]
ConflictingUrlsDirect(PackageName, String, String),
#[error("There are conflicting URLs for package `{0}`:\n- {1}\n- {2}")]
ConflictingUrlsTransitive(PackageName, String, String),
#[error("Package `{0}` attempted to resolve via URL: {1}. URL dependencies must be expressed as direct requirements or constraints. Consider adding `{0} @ {1}` to your dependencies or constraints file.")]
DisallowedUrl(PackageName, String),
#[error("There are conflicting editable requirements for package `{0}`:\n- {1}\n- {2}")]
ConflictingEditables(PackageName, String, String),
#[error(transparent)]
DistributionType(#[from] distribution_types::Error),
#[error("Failed to download `{0}`")]
Fetch(Box<BuiltDist>, #[source] uv_distribution::Error),
#[error("Failed to download and build `{0}`")]
FetchAndBuild(Box<SourceDist>, #[source] uv_distribution::Error),
#[error("Failed to read `{0}`")]
Read(Box<BuiltDist>, #[source] uv_distribution::Error),
// TODO(zanieb): Use `thiserror` in `InstalledDist` so we can avoid chaining `anyhow`
#[error("Failed to read metadata from installed package `{0}`")]
ReadInstalled(Box<InstalledDist>, #[source] anyhow::Error),
#[error("Failed to build `{0}`")]
Build(Box<SourceDist>, #[source] uv_distribution::Error),
#[error(transparent)]
NoSolution(#[from] NoSolutionError),
#[error("{package} {version} depends on itself")]
SelfDependency {
/// Package whose dependencies we want.
package: Box<PubGrubPackage>,
/// Version of the package for which we want the dependencies.
version: Box<Version>,
},
#[error("Attempted to construct an invalid version specifier")]
InvalidVersion(#[from] pep440_rs::VersionSpecifierBuildError),
#[error("In `--require-hashes` mode, all requirements must be pinned upfront with `==`, but found: `{0}`")]
UnhashedPackage(PackageName),
// TODO(konsti): Attach the distribution that contained the invalid requirement as error source.
#[error("Failed to parse requirements")]
DirectUrl(#[from] Box<ParsedUrlError>),
/// Something unexpected happened.
#[error("{0}")]
Failure(String),
}
impl<T> From<tokio::sync::mpsc::error::SendError<T>> for ResolveError {
/// Drop the value we want to send to not leak the private type we're sending.
/// The tokio error only says "channel closed", so we don't lose information.
fn from(_value: tokio::sync::mpsc::error::SendError<T>) -> Self {
Self::ChannelClosed
}
}
/// Given a [`DerivationTree`], collapse any [`External::FromDependencyOf`] incompatibilities
/// wrap an [`PubGrubPackageInner::Extra`] package.
fn collapse_extra_proxies(
derivation_tree: &mut DerivationTree<PubGrubPackage, Range<Version>, UnavailableReason>,
) {
match derivation_tree {
DerivationTree::External(_) => {}
DerivationTree::Derived(derived) => {
match (
Arc::make_mut(&mut derived.cause1),
Arc::make_mut(&mut derived.cause2),
) {
(
DerivationTree::External(External::FromDependencyOf(package, ..)),
ref mut cause,
) if matches!(&**package, PubGrubPackageInner::Extra { .. }) => {
collapse_extra_proxies(cause);
*derivation_tree = cause.clone();
}
(
ref mut cause,
DerivationTree::External(External::FromDependencyOf(package, ..)),
) if matches!(&**package, PubGrubPackageInner::Extra { .. }) => {
collapse_extra_proxies(cause);
*derivation_tree = cause.clone();
}
_ => {
collapse_extra_proxies(Arc::make_mut(&mut derived.cause1));
collapse_extra_proxies(Arc::make_mut(&mut derived.cause2));
}
}
}
}
}
impl From<pubgrub::error::PubGrubError<UvDependencyProvider>> for ResolveError {
fn from(value: pubgrub::error::PubGrubError<UvDependencyProvider>) -> Self {
match value {
// These are all never type variant that can never match, but never is experimental
pubgrub::error::PubGrubError::ErrorChoosingPackageVersion(_)
| pubgrub::error::PubGrubError::ErrorInShouldCancel(_)
| pubgrub::error::PubGrubError::ErrorRetrievingDependencies { .. } => {
unreachable!()
}
pubgrub::error::PubGrubError::Failure(inner) => Self::Failure(inner),
pubgrub::error::PubGrubError::NoSolution(mut derivation_tree) => {
collapse_extra_proxies(&mut derivation_tree);
Self::NoSolution(NoSolutionError {
derivation_tree,
// The following should be populated before display for the best error messages
available_versions: IndexMap::default(),
selector: None,
python_requirement: None,
index_locations: None,
unavailable_packages: FxHashMap::default(),
incomplete_packages: FxHashMap::default(),
})
}
pubgrub::error::PubGrubError::SelfDependency { package, version } => {
Self::SelfDependency {
package: Box::new(package),
version: Box::new(version),
}
}
}
}
}
/// A wrapper around [`pubgrub::error::PubGrubError::NoSolution`] that displays a resolution failure report.
#[derive(Debug)]
pub struct NoSolutionError {
derivation_tree: DerivationTree<PubGrubPackage, Range<Version>, UnavailableReason>,
available_versions: IndexMap<PubGrubPackage, BTreeSet<Version>>,
selector: Option<CandidateSelector>,
python_requirement: Option<PythonRequirement>,
index_locations: Option<IndexLocations>,
unavailable_packages: FxHashMap<PackageName, UnavailablePackage>,
incomplete_packages: FxHashMap<PackageName, BTreeMap<Version, IncompletePackage>>,
}
impl std::error::Error for NoSolutionError {}
impl std::fmt::Display for NoSolutionError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
// Write the derivation report.
let formatter = PubGrubReportFormatter {
available_versions: &self.available_versions,
python_requirement: self.python_requirement.as_ref(),
};
let report =
DefaultStringReporter::report_with_formatter(&self.derivation_tree, &formatter);
write!(f, "{report}")?;
// Include any additional hints.
for hint in formatter.hints(
&self.derivation_tree,
&self.selector,
&self.index_locations,
&self.unavailable_packages,
&self.incomplete_packages,
) {
write!(f, "\n\n{hint}")?;
}
Ok(())
}
}
impl NoSolutionError {
/// Update the available versions attached to the error using the given package version index.
///
/// Only packages used in the error's derivation tree will be retrieved.
#[must_use]
pub(crate) fn with_available_versions(
mut self,
python_requirement: &PythonRequirement,
visited: &FxHashSet<PackageName>,
package_versions: &FxOnceMap<PackageName, Arc<VersionsResponse>>,
) -> Self {
let mut available_versions = IndexMap::default();
for package in self.derivation_tree.packages() {
match &**package {
PubGrubPackageInner::Root(_) => {}
PubGrubPackageInner::Python(PubGrubPython::Installed) => {
available_versions.insert(
package.clone(),
BTreeSet::from([python_requirement.installed().deref().clone()]),
);
}
PubGrubPackageInner::Python(PubGrubPython::Target) => {
available_versions.insert(
package.clone(),
BTreeSet::from([python_requirement.target().deref().clone()]),
);
}
PubGrubPackageInner::Extra { .. } => {}
PubGrubPackageInner::Package { name, .. } => {
// Avoid including available versions for packages that exist in the derivation
// tree, but were never visited during resolution. We _may_ have metadata for
// these packages, but it's non-deterministic, and omitting them ensures that
// we represent the state of the resolver at the time of failure.
if visited.contains(name) {
if let Some(response) = package_versions.get(name) {
if let VersionsResponse::Found(ref version_maps) = *response {
for version_map in version_maps {
available_versions
.entry(package.clone())
.or_insert_with(BTreeSet::new)
.extend(
version_map.iter().map(|(version, _)| version.clone()),
);
}
}
}
}
}
}
}
self.available_versions = available_versions;
self
}
/// Update the candidate selector attached to the error.
#[must_use]
pub(crate) fn with_selector(mut self, selector: CandidateSelector) -> Self {
self.selector = Some(selector);
self
}
/// Update the index locations attached to the error.
#[must_use]
pub(crate) fn with_index_locations(mut self, index_locations: &IndexLocations) -> Self {
self.index_locations = Some(index_locations.clone());
self
}
/// Update the unavailable packages attached to the error.
#[must_use]
pub(crate) fn with_unavailable_packages(
mut self,
unavailable_packages: &DashMap<PackageName, UnavailablePackage>,
) -> Self {
let mut new = FxHashMap::default();
for package in self.derivation_tree.packages() {
if let PubGrubPackageInner::Package { name, .. } = &**package {
if let Some(reason) = unavailable_packages.get(name) {
new.insert(name.clone(), reason.clone());
}
}
}
self.unavailable_packages = new;
self
}
/// Update the incomplete packages attached to the error.
#[must_use]
pub(crate) fn with_incomplete_packages(
mut self,
incomplete_packages: &DashMap<PackageName, DashMap<Version, IncompletePackage>>,
) -> Self {
let mut new = FxHashMap::default();
for package in self.derivation_tree.packages() {
if let PubGrubPackageInner::Package { name, .. } = &**package {
if let Some(versions) = incomplete_packages.get(name) {
for entry in versions.iter() {
let (version, reason) = entry.pair();
new.entry(name.clone())
.or_insert_with(BTreeMap::default)
.insert(version.clone(), reason.clone());
}
}
}
}
self.incomplete_packages = new;
self
}
/// Update the Python requirements attached to the error.
#[must_use]
pub(crate) fn with_python_requirement(
mut self,
python_requirement: &PythonRequirement,
) -> Self {
self.python_requirement = Some(python_requirement.clone());
self
}
}