mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-10-03 07:14:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

uv-resolver: implement basic resolver forking

Browse source 
There are still some TODOs/FIXMEs here, but this makes represents a
chunk of the resolver refactoring to enable forking. We don't do any
merging of resolutions yet, so crucially, this code is broken when no
marker environment is provided. But when a marker environment is
provided, this should behave the same as a non-forking resolver. In
particular, `get_dependencies_forking` is just `get_dependencies`
whenever there's a marker environment.
This commit is contained in:
Andrew Gallant 2024-05-22 14:25:12 -04:00 • committed by Andrew Gallant
parent f5f330627b
commit 6f76a66510
1 changed files with 264 additions and 183 deletions
Download patch file Download diff file Expand all files Collapse all files

447
crates/uv-resolver/src/resolver/mod.rs
View file

@ -297,221 +297,236 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
) -> Result<ResolutionGraph, ResolveError> { ) -> Result<ResolutionGraph, ResolveError> {
let root = PubGrubPackage::from(PubGrubPackageInner::Root(self.project.clone())); let root = PubGrubPackage::from(PubGrubPackageInner::Root(self.project.clone()));
let mut prefetcher = BatchPrefetcher::default(); let mut prefetcher = BatchPrefetcher::default();
let mut state = SolveState { let state = SolveState {
pubgrub: State::init(root.clone(), MIN_VERSION.clone()), pubgrub: State::init(root.clone(), MIN_VERSION.clone()),
next: root, next: root,
pins: FilePins::default(), pins: FilePins::default(),
priorities: PubGrubPriorities::default(), priorities: PubGrubPriorities::default(),
added_dependencies: FxHashMap::default(), added_dependencies: FxHashMap::default(),
}; };
let mut forked_states = vec![state];
let mut resolutions = vec![];
debug!( debug!(
"Solving with target Python version {}", "Solving with target Python version {}",
self.python_requirement.target() self.python_requirement.target()
); );
loop { 'FORK: while let Some(mut state) = forked_states.pop() {
// Run unit propagation. loop {
state.pubgrub.unit_propagation(state.next)?; // Run unit propagation.
state.pubgrub.unit_propagation(state.next)?;
// Pre-visit all candidate packages, to allow metadata to be fetched in parallel. If // Pre-visit all candidate packages, to allow metadata to be fetched in parallel. If
// the dependency mode is direct, we only need to visit the root package. // the dependency mode is direct, we only need to visit the root package.
if self.dependency_mode.is_transitive() { if self.dependency_mode.is_transitive() {
Self::pre_visit( Self::pre_visit(
state.pubgrub.partial_solution.prioritized_packages(), state.pubgrub.partial_solution.prioritized_packages(),
&request_sink,
)?;
}
// Choose a package version.
let Some(highest_priority_pkg) = state
.pubgrub
.partial_solution
.pick_highest_priority_pkg(|package, _range| state.priorities.get(package))
else {
if enabled!(Level::DEBUG) {
prefetcher.log_tried_versions();
}
let selection = state.pubgrub.partial_solution.extract_solution();
resolutions.push(ResolutionGraph::from_state(
&selection,
&state.pins,
self.index.packages(),
self.index.distributions(),
&state.pubgrub,
&self.preferences,
)?);
continue 'FORK;
};
state.next = highest_priority_pkg;
prefetcher.version_tried(state.next.clone());
let term_intersection = state
.pubgrub
.partial_solution
.term_intersection_for_package(&state.next)
.ok_or_else(|| {
PubGrubError::Failure(
"a package was chosen but we don't have a term.".into(),
)
})?;
let decision = self.choose_version(
&state.next,
term_intersection.unwrap_positive(),
&mut state.pins,
visited,
&request_sink, &request_sink,
)?; )?;
}
// Choose a package version. // Pick the next compatible version.
let Some(highest_priority_pkg) = state let version = match decision {
.pubgrub None => {
.partial_solution debug!("No compatible version found for: {next}", next = state.next);
.pick_highest_priority_pkg(|package, _range| state.priorities.get(package))
else {
if enabled!(Level::DEBUG) {
prefetcher.log_tried_versions();
}
let selection = state.pubgrub.partial_solution.extract_solution();
return ResolutionGraph::from_state(
&selection,
&state.pins,
self.index.packages(),
self.index.distributions(),
&state.pubgrub,
&self.preferences,
);
};
state.next = highest_priority_pkg;
prefetcher.version_tried(state.next.clone()); let term_intersection = state
let term_intersection = state
.pubgrub
.partial_solution
.term_intersection_for_package(&state.next)
.ok_or_else(|| {
PubGrubError::Failure("a package was chosen but we don't have a term.".into())
})?;
let decision = self.choose_version(
&state.next,
term_intersection.unwrap_positive(),
&mut state.pins,
visited,
&request_sink,
)?;
// Pick the next compatible version.
let version = match decision {
None => {
debug!("No compatible version found for: {next}", next = state.next);
let term_intersection = state
.pubgrub
.partial_solution
.term_intersection_for_package(&state.next)
.expect("a package was chosen but we don't have a term.");
// Check if the decision was due to the package being unavailable
if let PubGrubPackageInner::Package { ref name, .. } = &*state.next {
if let Some(entry) = self.unavailable_packages.get(name) {
state
.pubgrub
.add_incompatibility(Incompatibility::custom_term(
state.next.clone(),
term_intersection.clone(),
UnavailableReason::Package(entry.clone()),
));
continue;
}
}
state
.pubgrub
.add_incompatibility(Incompatibility::no_versions(
state.next.clone(),
term_intersection.clone(),
));
continue;
}
Some(version) => version,
};
let version = match version {
ResolverVersion::Available(version) => version,
ResolverVersion::Unavailable(version, reason) => {
// Incompatible requires-python versions are special in that we track
// them as incompatible dependencies instead of marking the package version
// as unavailable directly
if let UnavailableVersion::IncompatibleDist(
IncompatibleDist::Source(IncompatibleSource::RequiresPython(
requires_python,
))
| IncompatibleDist::Wheel(IncompatibleWheel::RequiresPython(requires_python)),
) = reason
{
let python_version = requires_python
.iter()
.map(PubGrubSpecifier::try_from)
.fold_ok(Range::full(), |range, specifier| {
range.intersection(&specifier.into())
})?;
let package = &state.next;
for kind in [PubGrubPython::Installed, PubGrubPython::Target] {
state
.pubgrub
.add_incompatibility(Incompatibility::from_dependency(
package.clone(),
Range::singleton(version.clone()),
(
PubGrubPackage::from(PubGrubPackageInner::Python(kind)),
python_version.clone(),
),
));
}
state
.pubgrub .pubgrub
.partial_solution .partial_solution
.add_decision(state.next.clone(), version); .term_intersection_for_package(&state.next)
.expect("a package was chosen but we don't have a term.");
// Check if the decision was due to the package being unavailable
if let PubGrubPackageInner::Package { ref name, .. } = &*state.next {
if let Some(entry) = self.unavailable_packages.get(name) {
state
.pubgrub
.add_incompatibility(Incompatibility::custom_term(
state.next.clone(),
term_intersection.clone(),
UnavailableReason::Package(entry.clone()),
));
continue;
}
}
state
.pubgrub
.add_incompatibility(Incompatibility::no_versions(
state.next.clone(),
term_intersection.clone(),
));
continue; continue;
}; }
state Some(version) => version,
.pubgrub };
.add_incompatibility(Incompatibility::custom_version( let version = match version {
state.next.clone(), ResolverVersion::Available(version) => version,
version.clone(), ResolverVersion::Unavailable(version, reason) => {
UnavailableReason::Version(reason), // Incompatible requires-python versions are special in that we track
)); // them as incompatible dependencies instead of marking the package version
continue; // as unavailable directly
} if let UnavailableVersion::IncompatibleDist(
}; IncompatibleDist::Source(IncompatibleSource::RequiresPython(
requires_python,
))
| IncompatibleDist::Wheel(IncompatibleWheel::RequiresPython(
requires_python,
)),
) = reason
{
let python_version = requires_python
.iter()
.map(PubGrubSpecifier::try_from)
.fold_ok(Range::full(), |range, specifier| {
range.intersection(&specifier.into())
})?;
prefetcher.prefetch_batches( let package = &state.next;
&state.next, for kind in [PubGrubPython::Installed, PubGrubPython::Target] {
&version, state.pubgrub.add_incompatibility(
term_intersection.unwrap_positive(), Incompatibility::from_dependency(
&request_sink, package.clone(),
&self.index, Range::singleton(version.clone()),
&self.selector, (
)?; PubGrubPackage::from(PubGrubPackageInner::Python(kind)),
python_version.clone(),
self.on_progress(&state.next, &version); ),
),
if state );
.added_dependencies }
.entry(state.next.clone()) state
.or_default() .pubgrub
.insert(version.clone()) .partial_solution
{ .add_decision(state.next.clone(), version);
// Retrieve that package dependencies. continue;
let package = &state.next; };
let dependencies = match self.get_dependencies(
package,
&version,
&mut state.priorities,
&request_sink,
)? {
Dependencies::Unavailable(reason) => {
state state
.pubgrub .pubgrub
.add_incompatibility(Incompatibility::custom_version( .add_incompatibility(Incompatibility::custom_version(
package.clone(), state.next.clone(),
version.clone(), version.clone(),
UnavailableReason::Version(reason), UnavailableReason::Version(reason),
)); ));
continue; continue;
} }
Dependencies::Available(constraints)
if constraints
.iter()
.any(|(dependency, _)| dependency == package) =>
{
if enabled!(Level::DEBUG) {
prefetcher.log_tried_versions();
}
return Err(PubGrubError::SelfDependency {
package: package.clone(),
version: version.clone(),
}
.into());
}
Dependencies::Available(constraints) => constraints,
}; };
// Add that package and version if the dependencies are not problematic. prefetcher.prefetch_batches(
let dep_incompats = state.pubgrub.add_incompatibility_from_dependencies( &state.next,
package.clone(), &version,
version.clone(), term_intersection.unwrap_positive(),
dependencies, &request_sink,
); &self.index,
&self.selector,
)?;
state.pubgrub.partial_solution.add_version( self.on_progress(&state.next, &version);
package.clone(),
version, if state
dep_incompats, .added_dependencies
&state.pubgrub.incompatibility_store, .entry(state.next.clone())
); .or_default()
} else { .insert(version.clone())
{
// Retrieve that package dependencies.
let package = &state.next;
let forks = self.get_dependencies_forking(
package,
&version,
&mut state.priorities,
&request_sink,
)?;
for fork in forks {
let mut state = state.clone();
let dependencies = match fork {
Dependencies::Unavailable(reason) => {
state
.pubgrub
.add_incompatibility(Incompatibility::custom_version(
package.clone(),
version.clone(),
UnavailableReason::Version(reason),
));
forked_states.push(state);
continue;
}
Dependencies::Available(constraints)
if constraints
.iter()
.any(|(dependency, _)| dependency == package) =>
{
if enabled!(Level::DEBUG) {
prefetcher.log_tried_versions();
}
return Err(PubGrubError::SelfDependency {
package: package.clone(),
version: version.clone(),
}
.into());
}
Dependencies::Available(constraints) => constraints,
};
// Add that package and version if the dependencies are not problematic.
let dep_incompats = state.pubgrub.add_incompatibility_from_dependencies(
package.clone(),
version.clone(),
dependencies,
);
state.pubgrub.partial_solution.add_version(
package.clone(),
version.clone(),
dep_incompats,
&state.pubgrub.incompatibility_store,
);
forked_states.push(state);
}
continue 'FORK;
}
// `dep_incompats` are already in `incompatibilities` so we know there are not satisfied // `dep_incompats` are already in `incompatibilities` so we know there are not satisfied
// terms and can add the decision directly. // terms and can add the decision directly.
state state
@ -520,6 +535,11 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
.add_decision(state.next.clone(), version); .add_decision(state.next.clone(), version);
} }
} }
// This unwrap is okay because every code path above leads to at least
// one resolution being pushed.
//
// TODO: Implement merging of resolutions.
Ok(resolutions.pop().unwrap())
} }
/// Visit a [`PubGrubPackage`] prior to selection. This should be called on a [`PubGrubPackage`] /// Visit a [`PubGrubPackage`] prior to selection. This should be called on a [`PubGrubPackage`]
@ -796,6 +816,67 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
} }
} }
/// Given a candidate package and version, return its dependencies.
#[instrument(skip_all, fields(%package, %version))]
fn get_dependencies_forking(
&self,
package: &PubGrubPackage,
version: &Version,
priorities: &mut PubGrubPriorities,
request_sink: &Sender<Request>,
) -> Result<Vec<Dependencies>, ResolveError> {
type Dep = (PubGrubPackage, Range<Version>);
let result = self.get_dependencies(package, version, priorities, request_sink);
if self.markers.is_some() {
return result.map(|deps| vec![deps]);
}
let deps: Vec<Dep> = match result? {
Dependencies::Available(deps) => deps,
Dependencies::Unavailable(err) => return Ok(vec![Dependencies::Unavailable(err)]),
};
let mut by_grouping: FxHashMap<&PackageName, FxHashMap<&Range<Version>, Vec<&Dep>>> =
FxHashMap::default();
for dep in &deps {
let (ref pkg, ref range) = *dep;
let name = match &**pkg {
// A root can never be a dependency of another package, and a `Python` pubgrub
// package is never returned by `get_dependencies`. So these cases never occur.
PubGrubPackageInner::Root(_) | PubGrubPackageInner::Python(_) => unreachable!(),
PubGrubPackageInner::Package { ref name, .. }
| PubGrubPackageInner::Extra { ref name, .. } => name,
};
by_grouping
.entry(name)
.or_default()
.entry(range)
.or_default()
.push(dep);
}
let mut forks: Vec<Vec<Dep>> = vec![vec![]];
for (_, groups) in by_grouping {
if groups.len() <= 1 {
for deps in groups.into_values() {
for fork in &mut forks {
fork.extend(deps.iter().map(|dep| (*dep).clone()));
}
}
} else {
let mut new_forks: Vec<Vec<Dep>> = vec![];
for deps in groups.into_values() {
let mut new_forks_for_group = forks.clone();
for fork in &mut new_forks_for_group {
fork.extend(deps.iter().map(|dep| (*dep).clone()));
}
new_forks.extend(new_forks_for_group);
}
forks = new_forks;
}
}
Ok(forks.into_iter().map(Dependencies::Available).collect())
}
/// Given a candidate package and version, return its dependencies. /// Given a candidate package and version, return its dependencies.
#[instrument(skip_all, fields(%package, %version))] #[instrument(skip_all, fields(%package, %version))]
fn get_dependencies( fn get_dependencies(