mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Avoid double-resolving during pip-install (#610)

Browse source 
## Summary

At present, when performing a `pip-install`, we first do a resolution,
then take the set of requirements and basically run them through our
`pip-sync`, which itself includes re-resolving the dependencies to get a
specific `Dist` for each package. (E.g., the set of requirements might
say `flask==3.0.0`, but the installer needs a specific _wheel_ or source
distribution to install.)

This PR removes this second resolution by exposing the set of pinned
packages from the resolution. The main challenge here is that we have an
optimization in the resolver such that we let the resolver read metadata
from an incompatible wheel as long as a source distribution exists for a
given package. This lets us avoid building source distributions in the
resolver under the assumption that we'll be able to install the package
later on, if needed. As such, the resolver now needs to track the
resolution and installation filenames separately.
This commit is contained in:
Charlie Marsh 2023-12-12 12:29:09 -05:00 • committed by GitHub
parent a0b3815d84
commit c764155988
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 289 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

53
crates/puffin-cli/src/commands/pip_install.rs
View file

@ -18,14 +18,12 @@ use puffin_dispatch::BuildDispatch;
use puffin_installer::{Downloader, InstallPlan, Reinstall, SitePackages};
use puffin_interpreter::Virtualenv;
use puffin_resolver::{
Graph, Manifest, PreReleaseMode, ResolutionMode, ResolutionOptions, Resolver,
Graph, Manifest, PreReleaseMode, Resolution, ResolutionMode, ResolutionOptions, Resolver,
};
use puffin_traits::OnceMap;
use pypi_types::IndexUrls;
use crate::commands::reporters::{
DownloadReporter, FinderReporter, InstallReporter, ResolverReporter,
};
use crate::commands::reporters::{DownloadReporter, InstallReporter, ResolverReporter};
use crate::commands::{elapsed, ChangeEvent, ChangeEventKind, ExitStatus};
use crate::printer::Printer;
use crate::requirements::{ExtrasSpecification, RequirementsSource, RequirementsSpecification};
@ -84,7 +82,7 @@ pub(crate) async fn pip_install(
// Sync the environment.
install(
&resolution.requirements(),
&resolution.into(),
reinstall,
link_mode,
index_urls,
@ -241,7 +239,7 @@ async fn resolve(
/// Install a set of requirements into the current environment.
#[allow(clippy::too_many_arguments)]
async fn install(
requirements: &[Requirement],
resolution: &Resolution,
reinstall: &Reinstall,
link_mode: LinkMode,
index_urls: IndexUrls,
@ -264,7 +262,7 @@ async fn install(
reinstalls,
extraneous: _,
} = InstallPlan::from_requirements(
requirements,
&resolution.requirements(),
reinstall,
&index_urls,
cache,
@ -276,13 +274,13 @@ async fn install(
// Nothing to do.
if remote.is_empty() && local.is_empty() && reinstalls.is_empty() {
let s = if requirements.len() == 1 { "" } else { "s" };
let s = if resolution.len() == 1 { "" } else { "s" };
writeln!(
printer,
"{}",
format!(
"Audited {} in {}",
format!("{} package{}", requirements.len(), s).bold(),
format!("{} package{}", resolution.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
@ -297,34 +295,15 @@ async fn install(
.build();
// Resolve any registry-based requirements.
// TODO(charlie): We should be able to reuse the resolution from the `resolve` step. All the
// responses will be cached, so this isn't _terrible_, but it is wasteful. (Note that the
// distributions chosen when resolving won't necessarily be the same as those chosen by the
// `DistFinder`, since we allow the use of incompatible wheels when resolving, as long as a
// source distribution is present.)
let remote = if remote.is_empty() {
Vec::new()
} else {
let start = std::time::Instant::now();
let wheel_finder = puffin_resolver::DistFinder::new(&tags, &client, venv.interpreter())
.with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64));
let resolution = wheel_finder.resolve(&remote).await?;
let s = if resolution.len() == 1 { "" } else { "s" };
writeln!(
printer,
"{}",
format!(
"Resolved {} in {}",
format!("{} package{}", resolution.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
resolution.into_distributions().collect::<Vec<_>>()
};
let remote = remote
.iter()
.map(|dist| {
resolution
.get(&dist.name)
.cloned()
.expect("Resolution should contain all packages")
})
.collect::<Vec<_>>();
// Download, build, and unzip any missing distributions.
let wheels = if remote.is_empty() {

7
crates/puffin-cli/tests/pip_install.rs
View file

@ -78,7 +78,6 @@ fn install_package() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 7 packages in [TIME]
Downloaded 7 packages in [TIME]
Installed 7 packages in [TIME]
+ blinker==1.7.0
@ -126,7 +125,6 @@ fn install_requirements_txt() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 7 packages in [TIME]
Downloaded 7 packages in [TIME]
Installed 7 packages in [TIME]
+ blinker==1.7.0
@ -204,7 +202,6 @@ fn respect_installed() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 7 packages in [TIME]
Downloaded 7 packages in [TIME]
Installed 7 packages in [TIME]
+ blinker==1.7.0
@ -273,7 +270,6 @@ fn respect_installed() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
- flask==2.3.2
@ -307,7 +303,6 @@ fn respect_installed() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
- flask==2.3.3
@ -349,7 +344,6 @@ fn allow_incompatibilities() -> Result<()> {
----- stderr -----
Resolved 7 packages in [TIME]
Resolved 7 packages in [TIME]
Downloaded 7 packages in [TIME]
Installed 7 packages in [TIME]
+ blinker==1.7.0
@ -388,7 +382,6 @@ fn allow_incompatibilities() -> Result<()> {
----- stderr -----
Resolved 2 packages in [TIME]
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
- jinja2==3.1.2

98
crates/puffin-resolver/src/candidate_selector.rs
View file

@ -1,14 +1,16 @@
use pubgrub::range::Range;
use rustc_hash::FxHashMap;
use distribution_types::Dist;
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_normalize::PackageName;
use pypi_types::IndexUrl;
use crate::file::DistFile;
use crate::prerelease_mode::PreReleaseStrategy;
use crate::pubgrub::PubGrubVersion;
use crate::resolution_mode::ResolutionStrategy;
use crate::version_map::VersionMap;
use crate::version_map::{ResolvableFile, VersionMap};
use crate::{Manifest, ResolutionOptions};
#[derive(Debug)]
@ -76,22 +78,18 @@ enum AllowPreRelease {
impl CandidateSelector {
/// Select a [`Candidate`] from a set of candidate versions and files.
pub(crate) fn select(
&self,
package_name: &PackageName,
pub(crate) fn select<'a>(
&'a self,
package_name: &'a PackageName,
range: &Range<PubGrubVersion>,
version_map: &VersionMap,
) -> Option<Candidate> {
version_map: &'a VersionMap,
) -> Option<Candidate<'a>> {
// If the package has a preference (e.g., an existing version from an existing lockfile),
// and the preference satisfies the current range, use that.
if let Some(version) = self.preferences.get(package_name) {
if range.contains(version) {
if let Some(file) = version_map.get(version) {
return Some(Candidate {
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
return Some(Candidate::new(package_name, version, file));
}
}
}
@ -150,15 +148,15 @@ impl CandidateSelector {
/// Select the first-matching [`Candidate`] from a set of candidate versions and files,
/// preferring wheels over sdists.
fn select_candidate<'a>(
versions: impl Iterator<Item = (&'a PubGrubVersion, &'a DistFile)>,
package_name: &PackageName,
versions: impl Iterator<Item = (&'a PubGrubVersion, ResolvableFile<'a>)>,
package_name: &'a PackageName,
range: &Range<PubGrubVersion>,
allow_prerelease: AllowPreRelease,
) -> Option<Candidate> {
) -> Option<Candidate<'a>> {
#[derive(Debug)]
enum PreReleaseCandidate<'a> {
NotNecessary,
IfNecessary(&'a PubGrubVersion, &'a DistFile),
IfNecessary(&'a PubGrubVersion, ResolvableFile<'a>),
}
let mut prerelease = None;
@ -169,11 +167,7 @@ impl CandidateSelector {
AllowPreRelease::Yes => {
// If pre-releases are allowed, treat them equivalently
// to stable distributions.
return Some(Candidate {
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
return Some(Candidate::new(package_name, version, file));
}
AllowPreRelease::IfNecessary => {
// If pre-releases are allowed as a fallback, store the
@ -195,32 +189,66 @@ impl CandidateSelector {
// Always return the first-matching stable distribution.
if range.contains(version) {
return Some(Candidate {
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
});
return Some(Candidate::new(package_name, version, file));
}
}
}
match prerelease {
None => None,
Some(PreReleaseCandidate::NotNecessary) => None,
Some(PreReleaseCandidate::IfNecessary(version, file)) => Some(Candidate {
package_name: package_name.clone(),
version: version.clone(),
file: file.clone(),
}),
Some(PreReleaseCandidate::IfNecessary(version, file)) => {
Some(Candidate::new(package_name, version, file))
}
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct Candidate {
pub(crate) struct Candidate<'a> {
/// The name of the package.
pub(crate) package_name: PackageName,
name: &'a PackageName,
/// The version of the package.
pub(crate) version: PubGrubVersion,
/// The file of the package.
pub(crate) file: DistFile,
version: &'a PubGrubVersion,
/// The file to use for resolving and installing the package.
file: ResolvableFile<'a>,
}
impl<'a> Candidate<'a> {
fn new(name: &'a PackageName, version: &'a PubGrubVersion, file: ResolvableFile<'a>) -> Self {
Self {
name,
version,
file,
}
}
/// Return the name of the package.
pub(crate) fn name(&self) -> &PackageName {
self.name
}
/// Return the version of the package.
pub(crate) fn version(&self) -> &PubGrubVersion {
self.version
}
/// Return the [`DistFile`] to use when resolving the package.
pub(crate) fn resolve(&self) -> &DistFile {
self.file.resolve()
}
/// Return the [`DistFile`] to use when installing the package.
pub(crate) fn install(&self) -> &DistFile {
self.file.install()
}
/// Return the [`Dist`] to use when resolving the candidate.
pub(crate) fn into_distribution(self, index: IndexUrl) -> Dist {
Dist::from_registry(
self.name().clone(),
self.version().clone().into(),
self.resolve().clone().into(),
index,
)
}
}

3
crates/puffin-resolver/src/lib.rs
View file

@ -3,7 +3,7 @@ pub use finder::{DistFinder, Reporter as FinderReporter};
pub use manifest::Manifest;
pub use prerelease_mode::PreReleaseMode;
pub use pubgrub::PubGrubReportFormatter;
pub use resolution::Graph;
pub use resolution::{Graph, Resolution};
pub use resolution_mode::ResolutionMode;
pub use resolution_options::ResolutionOptions;
pub use resolver::{
@ -15,6 +15,7 @@ mod error;
mod file;
mod finder;
mod manifest;
mod pins;
mod prerelease_mode;
mod pubgrub;
mod resolution;

30
crates/puffin-resolver/src/pins.rs Normal file
View file

@ -0,0 +1,30 @@
use crate::candidate_selector::Candidate;
use puffin_normalize::PackageName;
use pypi_types::{File, IndexUrl};
use rustc_hash::FxHashMap;
/// A set of package versions pinned to specific files.
///
/// For example, given `Flask==3.0.0`, the [`FilePins`] would contain a mapping from `Flask` to
/// `3.0.0` to the specific wheel or source distribution archive that was pinned for that version.
#[derive(Debug, Default)]
pub(crate) struct FilePins(FxHashMap<PackageName, FxHashMap<pep440_rs::Version, (IndexUrl, File)>>);
impl FilePins {
/// Pin a candidate package.
pub(crate) fn insert(&mut self, candidate: &Candidate, index: &IndexUrl) {
self.0.entry(candidate.name().clone()).or_default().insert(
candidate.version().clone().into(),
(index.clone(), candidate.install().clone().into()),
);
}
/// Return the pinned file for the given package name and version, if it exists.
pub(crate) fn get(
&self,
name: &PackageName,
version: &pep440_rs::Version,
) -> Option<&(IndexUrl, File)> {
self.0.get(name)?.get(version)
}
}

135
crates/puffin-resolver/src/resolution.rs
View file

@ -2,6 +2,7 @@ use std::hash::BuildHasherDefault;
use anyhow::Result;
use colored::Colorize;
use itertools::Itertools;
use petgraph::visit::EdgeRef;
use petgraph::Direction;
use pubgrub::range::Range;
@ -15,8 +16,8 @@ use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_normalize::PackageName;
use puffin_traits::OnceMap;
use pypi_types::{File, IndexUrl};
use crate::pins::FilePins;
use crate::pubgrub::{PubGrubPackage, PubGrubPriority, PubGrubVersion};
use crate::ResolveError;
@ -49,6 +50,15 @@ impl Resolution {
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Return the set of [`Requirement`]s that this resolution represents.
pub fn requirements(&self) -> Vec<Requirement> {
self.0
.values()
.sorted_by_key(|package| package.name())
.map(as_requirement)
.collect()
}
}
/// A complete resolution graph in which every node represents a pinned package and every edge
@ -58,9 +68,9 @@ pub struct Graph(pub petgraph::graph::Graph<Dist, Range<PubGrubVersion>, petgrap
impl Graph {
/// Create a new graph from the resolved `PubGrub` state.
pub fn from_state(
pub(crate) fn from_state(
selection: &SelectedDependencies<PubGrubPackage, PubGrubVersion>,
pins: &FxHashMap<PackageName, FxHashMap<Version, (IndexUrl, File)>>,
pins: &FilePins,
redirects: &OnceMap<Url, Url>,
state: &State<PubGrubPackage, Range<PubGrubVersion>, PubGrubPriority>,
) -> Result<Self, ResolveError> {
@ -76,9 +86,8 @@ impl Graph {
PubGrubPackage::Package(package_name, None, None) => {
let version = Version::from(version.clone());
let (index, file) = pins
.get(package_name)
.and_then(|versions| versions.get(&version))
.unwrap()
.get(package_name, &version)
.expect("Every package should be pinned")
.clone();
let pinned_package =
Dist::from_registry(package_name.clone(), version, file, index);
@ -139,6 +148,7 @@ impl Graph {
self.0.node_count() == 0
}
/// Return the set of [`Requirement`]s that this graph represents.
pub fn requirements(&self) -> Vec<Requirement> {
// Collect and sort all packages.
let mut nodes = self
@ -149,54 +159,7 @@ impl Graph {
nodes.sort_unstable_by_key(|(_, package)| package.name());
self.0
.node_indices()
.map(|node| match &self.0[node] {
Dist::Built(BuiltDist::Registry(wheel)) => Requirement {
name: wheel.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from(
VersionSpecifier::equals_version(wheel.version.clone()),
))),
marker: None,
},
Dist::Built(BuiltDist::DirectUrl(wheel)) => Requirement {
name: wheel.filename.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())),
marker: None,
},
Dist::Built(BuiltDist::Path(wheel)) => Requirement {
name: wheel.filename.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Registry(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from(
VersionSpecifier::equals_version(sdist.version.clone()),
))),
marker: None,
},
Dist::Source(SourceDist::DirectUrl(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Git(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Path(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
})
.map(|node| as_requirement(&self.0[node]))
.collect()
}
@ -249,3 +212,67 @@ impl std::fmt::Display for Graph {
Ok(())
}
}
impl From<Graph> for Resolution {
fn from(graph: Graph) -> Self {
Self(
graph
.0
.node_indices()
.map(|node| (graph.0[node].name().clone(), graph.0[node].clone()))
.collect(),
)
}
}
/// Create a [`Requirement`] from a [`Dist`].
fn as_requirement(dist: &Dist) -> Requirement {
match dist {
Dist::Built(BuiltDist::Registry(wheel)) => Requirement {
name: wheel.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from(
VersionSpecifier::equals_version(wheel.version.clone()),
))),
marker: None,
},
Dist::Built(BuiltDist::DirectUrl(wheel)) => Requirement {
name: wheel.filename.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())),
marker: None,
},
Dist::Built(BuiltDist::Path(wheel)) => Requirement {
name: wheel.filename.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Registry(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from(
VersionSpecifier::equals_version(sdist.version.clone()),
))),
marker: None,
},
Dist::Source(SourceDist::DirectUrl(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Git(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
Dist::Source(SourceDist::Path(sdist)) => Requirement {
name: sdist.name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())),
marker: None,
},
}
}

106
crates/puffin-resolver/src/resolver.rs
View file

@ -8,18 +8,17 @@ use anyhow::Result;
use chrono::{DateTime, Utc};
use futures::channel::mpsc::UnboundedReceiver;
use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt};
use rustc_hash::{FxHashMap, FxHashSet};
use pubgrub::error::PubGrubError;
use pubgrub::range::Range;
use pubgrub::solver::{Incompatibility, State};
use pubgrub::type_aliases::DependencyConstraints;
use rustc_hash::{FxHashMap, FxHashSet};
use tokio::select;
use tracing::{debug, trace};
use url::Url;
use distribution_filename::WheelFilename;
use distribution_types::{BuiltDist, Dist, Identifier, Metadata, SourceDist, VersionOrUrl};
use distribution_types::{BuiltDist, Dist, Metadata, SourceDist, VersionOrUrl};
use pep508_rs::{MarkerEnvironment, Requirement};
use platform_tags::Tags;
use puffin_cache::CanonicalUrl;
@ -27,11 +26,12 @@ use puffin_client::RegistryClient;
use puffin_distribution::{DistributionDatabase, DistributionDatabaseError};
use puffin_normalize::{ExtraName, PackageName};
use puffin_traits::{BuildContext, OnceMap};
use pypi_types::{File, IndexUrl, Metadata21};
use pypi_types::{IndexUrl, Metadata21};
use crate::candidate_selector::CandidateSelector;
use crate::error::ResolveError;
use crate::manifest::Manifest;
use crate::pins::FilePins;
use crate::pubgrub::{
PubGrubDependencies, PubGrubPackage, PubGrubPriorities, PubGrubVersion, MIN_VERSION,
};
@ -270,7 +270,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
// Keep track of the packages for which we've requested metadata.
let index = Index::default();
let mut pins = FxHashMap::default();
let mut pins = FilePins::default();
let mut priorities = PubGrubPriorities::default();
// Start the solve.
@ -342,14 +342,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
// Retrieve that package dependencies.
let package = &next;
let dependencies = match self
.get_dependencies(
package,
&version,
&mut pins,
&mut priorities,
&index,
request_sink,
)
.get_dependencies(package, &version, &mut priorities, &index, request_sink)
.await?
{
Dependencies::Unknown => {
@ -458,15 +451,10 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
if self
.index
.distributions
.register(candidate.file.sha256())
.register(candidate.resolve().sha256())
.await
{
let distribution = Dist::from_registry(
candidate.package_name,
candidate.version.into(),
candidate.file.into(),
index.clone(),
);
let distribution = candidate.into_distribution(index.clone());
request_sink.unbounded_send(Request::Dist(distribution))?;
}
}
@ -479,7 +467,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
&self,
package: &PubGrubPackage,
range: &Range<PubGrubVersion>,
pins: &mut FxHashMap<PackageName, FxHashMap<pep440_rs::Version, (IndexUrl, File)>>,
pins: &mut FilePins,
request_sink: &futures::channel::mpsc::UnboundedSender<Request>,
) -> Result<Option<PubGrubVersion>, ResolveError> {
return match package {
@ -506,7 +494,8 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
}
} else {
// Otherwise, assume this is a source distribution.
let entry = self.index.distributions.wait(&url.distribution_id()).await;
let dist = PubGrubDistribution::from_url(package_name, url);
let entry = self.index.distributions.wait(&dist.package_id()).await;
let metadata = entry.value();
let version = PubGrubVersion::from(metadata.version.clone());
if range.contains(&version) {
@ -532,35 +521,25 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
debug!(
"Selecting: {}=={} ({})",
candidate.package_name,
candidate.version,
candidate.file.filename()
candidate.name(),
candidate.version(),
candidate.resolve().filename()
);
// We want to return a package pinned to a specific version; but we _also_ want to
// store the exact file that we selected to satisfy that version.
pins.entry(candidate.package_name.clone())
.or_default()
.insert(
candidate.version.clone().into(),
(index.clone(), candidate.file.clone().into()),
);
pins.insert(&candidate, index);
let version = candidate.version.clone();
let version = candidate.version().clone();
// Emit a request to fetch the metadata for this version.
if self
.index
.distributions
.register(candidate.file.sha256())
.register(candidate.resolve().sha256())
.await
{
let distribution = Dist::from_registry(
candidate.package_name,
candidate.version.into(),
candidate.file.into(),
index.clone(),
);
let distribution = candidate.into_distribution(index.clone());
request_sink.unbounded_send(Request::Dist(distribution))?;
}
@ -574,7 +553,6 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
&self,
package: &PubGrubPackage,
version: &PubGrubVersion,
pins: &mut FxHashMap<PackageName, FxHashMap<pep440_rs::Version, (IndexUrl, File)>>,
priorities: &mut PubGrubPriorities,
index: &Index,
request_sink: &futures::channel::mpsc::UnboundedSender<Request>,
@ -610,14 +588,11 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
PubGrubPackage::Package(package_name, extra, url) => {
// Wait for the metadata to be available.
let entry = match url {
Some(url) => self.index.distributions.wait(&url.distribution_id()).await,
None => {
let versions = pins.get(package_name).unwrap();
let (_index, file) = versions.get(version.into()).unwrap();
self.index.distributions.wait(&file.distribution_id()).await
}
let dist = match url {
Some(url) => PubGrubDistribution::from_url(package_name, url),
None => PubGrubDistribution::from_registry(package_name, version),
};
let entry = self.index.distributions.wait(&dist.package_id()).await;
let metadata = entry.value();
let mut constraints = PubGrubDependencies::try_from_requirements(
@ -664,20 +639,19 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
match response? {
Response::Package(package_name, index, version_map) => {
trace!("Received package metadata for: {package_name}");
self.index.packages.done(package_name, (index, version_map));
}
Response::Dist(Dist::Built(distribution), metadata, ..) => {
trace!("Received built distribution metadata for: {distribution}");
self.index
.distributions
.done(distribution.distribution_id(), metadata);
.done(distribution.package_id(), metadata);
}
Response::Dist(Dist::Source(distribution), metadata, precise) => {
trace!("Received source distribution metadata for: {distribution}");
self.index
.distributions
.done(distribution.distribution_id(), metadata);
.done(distribution.package_id(), metadata);
if let Some(precise) = precise {
match distribution {
SourceDist::DirectUrl(sdist) => {
@ -862,3 +836,35 @@ enum Dependencies {
/// Container for all available package versions.
Known(DependencyConstraints<PubGrubPackage, Range<PubGrubVersion>>),
}
#[derive(Debug)]
enum PubGrubDistribution<'a> {
Registry(&'a PackageName, &'a PubGrubVersion),
Url(&'a PackageName, &'a Url),
}
impl<'a> PubGrubDistribution<'a> {
fn from_registry(name: &'a PackageName, version: &'a PubGrubVersion) -> Self {
Self::Registry(name, version)
}
fn from_url(name: &'a PackageName, url: &'a Url) -> Self {
Self::Url(name, url)
}
}
impl Metadata for PubGrubDistribution<'_> {
fn name(&self) -> &PackageName {
match self {
Self::Registry(name, _) => name,
Self::Url(name, _) => name,
}
}
fn version_or_url(&self) -> VersionOrUrl {
match self {
Self::Registry(_, version) => VersionOrUrl::Version((*version).into()),
Self::Url(_, url) => VersionOrUrl::Url(url),
}
}
}

49
crates/puffin-resolver/src/version_map.rs
View file

@ -7,6 +7,7 @@ use tracing::warn;
use distribution_filename::DistFilename;
use pep508_rs::MarkerEnvironment;
use platform_tags::{TagPriority, Tags};
use puffin_client::SimpleMetadata;
use puffin_interpreter::Interpreter;
use puffin_macros::warn_once;
use puffin_normalize::PackageName;
@ -15,7 +16,6 @@ use pypi_types::Yanked;
use crate::file::{DistFile, SdistFile, WheelFile};
use crate::pubgrub::PubGrubVersion;
use crate::yanks::AllowedYanks;
use puffin_client::SimpleMetadata;
/// A map from versions to distributions.
#[derive(Debug, Default)]
@ -117,12 +117,14 @@ impl VersionMap {
}
/// Return the [`DistFile`] for the given version, if any.
pub(crate) fn get(&self, version: &PubGrubVersion) -> Option<&DistFile> {
self.0.get(version).and_then(|file| file.get())
pub(crate) fn get(&self, version: &PubGrubVersion) -> Option<ResolvableFile> {
self.0.get(version).and_then(PrioritizedDistribution::get)
}
/// Return an iterator over the versions and distributions.
pub(crate) fn iter(&self) -> impl DoubleEndedIterator<Item = (&PubGrubVersion, &DistFile)> {
pub(crate) fn iter(
&self,
) -> impl DoubleEndedIterator<Item = (&PubGrubVersion, ResolvableFile)> {
self.0
.iter()
.filter_map(|(version, file)| Some((version, file.get()?)))
@ -190,22 +192,53 @@ impl PrioritizedDistribution {
}
/// Return the highest-priority distribution for the package version, if any.
fn get(&self) -> Option<&DistFile> {
fn get(&self) -> Option<ResolvableFile> {
match (
&self.compatible_wheel,
&self.source,
&self.incompatible_wheel,
) {
// Prefer the highest-priority, platform-compatible wheel.
(Some((file, _)), _, _) => Some(file),
(Some((wheel, _)), _, _) => Some(ResolvableFile::CompatibleWheel(wheel)),
// If we have a source distribution and an incompatible wheel, return the wheel.
// We assume that all distributions have the same metadata for a given package version.
// If a source distribution exists, we assume we can build it, but using the wheel is
// faster.
(_, Some(_), Some(file)) => Some(file),
(_, Some(sdist), Some(wheel)) => Some(ResolvableFile::IncompatibleWheel(sdist, wheel)),
// Otherwise, return the source distribution.
(_, Some(file), _) => Some(file),
(_, Some(sdist), _) => Some(ResolvableFile::SourceDist(sdist)),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub(crate) enum ResolvableFile<'a> {
/// The distribution should be resolved and installed using a source distribution.
SourceDist(&'a DistFile),
/// The distribution should be resolved and installed using a wheel distribution.
CompatibleWheel(&'a DistFile),
/// The distribution should be resolved using an incompatible wheel distribution, but
/// installed using a source distribution.
IncompatibleWheel(&'a DistFile, &'a DistFile),
}
impl<'a> ResolvableFile<'a> {
/// Return the [`DistFile`] to use during resolution.
pub(crate) fn resolve(&self) -> &DistFile {
match self {
ResolvableFile::SourceDist(sdist) => sdist,
ResolvableFile::CompatibleWheel(wheel) => wheel,
ResolvableFile::IncompatibleWheel(_, wheel) => wheel,
}
}
/// Return the [`DistFile`] to use during installation.
pub(crate) fn install(&self) -> &DistFile {
match self {
ResolvableFile::SourceDist(sdist) => sdist,
ResolvableFile::CompatibleWheel(wheel) => wheel,
ResolvableFile::IncompatibleWheel(sdist, _) => sdist,
}
}
}