mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Use operations API in pip compile (#4493)

Browse source 
## Summary

Closes https://github.com/astral-sh/uv/issues/4235.
This commit is contained in:
Charlie Marsh 2024-06-25 01:20:03 +03:00 • committed by GitHub
parent 9905521957
commit 10ec9c9d0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 157 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

66
crates/uv-resolver/src/resolution/display.rs
View file

@ -5,7 +5,8 @@ use petgraph::visit::EdgeRef;
use petgraph::Direction;
use rustc_hash::{FxBuildHasher, FxHashMap};
use distribution_types::{Name, SourceAnnotations};
use distribution_types::{Name, SourceAnnotation, SourceAnnotations};
use pep508_rs::MarkerEnvironment;
use uv_normalize::PackageName;
use crate::resolution::RequirementsTxtDist;
@ -17,6 +18,8 @@ use crate::ResolutionGraph;
pub struct DisplayResolutionGraph<'a> {
/// The underlying graph.
resolution: &'a ResolutionGraph,
/// The marker environment, used to determine the markers that apply to each package.
marker_env: Option<&'a MarkerEnvironment>,
/// The packages to exclude from the output.
no_emit_packages: &'a [PackageName],
/// Whether to include hashes in the output.
@ -31,21 +34,19 @@ pub struct DisplayResolutionGraph<'a> {
/// The style of annotation comments, used to indicate the dependencies that requested each
/// package.
annotation_style: AnnotationStyle,
/// External sources for each package: requirements, constraints, and overrides.
sources: SourceAnnotations,
}
impl<'a> From<&'a ResolutionGraph> for DisplayResolutionGraph<'a> {
fn from(resolution: &'a ResolutionGraph) -> Self {
Self::new(
resolution,
None,
&[],
false,
false,
true,
false,
AnnotationStyle::default(),
SourceAnnotations::default(),
)
}
}
@ -55,23 +56,23 @@ impl<'a> DisplayResolutionGraph<'a> {
#[allow(clippy::fn_params_excessive_bools, clippy::too_many_arguments)]
pub fn new(
underlying: &'a ResolutionGraph,
marker_env: Option<&'a MarkerEnvironment>,
no_emit_packages: &'a [PackageName],
show_hashes: bool,
include_extras: bool,
include_annotations: bool,
include_index_annotation: bool,
annotation_style: AnnotationStyle,
sources: SourceAnnotations,
) -> DisplayResolutionGraph<'a> {
Self {
resolution: underlying,
marker_env,
no_emit_packages,
show_hashes,
include_extras,
include_annotations,
include_index_annotation,
annotation_style,
sources,
}
}
}
@ -79,6 +80,57 @@ impl<'a> DisplayResolutionGraph<'a> {
/// Write the graph in the `{name}=={version}` format of requirements.txt that pip uses.
impl std::fmt::Display for DisplayResolutionGraph<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// Determine the annotation sources for each package.
let sources = if self.include_annotations {
let mut sources = SourceAnnotations::default();
for requirement in self
.resolution
.requirements
.iter()
.filter(|requirement| requirement.evaluate_markers(self.marker_env, &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Requirement(origin.clone()),
);
}
}
for requirement in self
.resolution
.constraints
.requirements()
.filter(|requirement| requirement.evaluate_markers(self.marker_env, &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Constraint(origin.clone()),
);
}
}
for requirement in self
.resolution
.overrides
.requirements()
.filter(|requirement| requirement.evaluate_markers(self.marker_env, &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Override(origin.clone()),
);
}
}
sources
} else {
SourceAnnotations::default()
};
// Reduce the graph, such that all nodes for a single package are combined, regardless of
// the extras.
//
@ -171,7 +223,7 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> {
// Include all external sources (e.g., requirements files).
let default = BTreeSet::default();
let source = self.sources.get(node.name()).unwrap_or(&default);
let source = sources.get(node.name()).unwrap_or(&default);
match self.annotation_style {
AnnotationStyle::Line => match edges.as_slice() {

33
crates/uv-resolver/src/resolution/graph.rs
View file

@ -10,7 +10,8 @@ use distribution_types::{
};
use pep440_rs::{Version, VersionSpecifier};
use pep508_rs::{MarkerEnvironment, MarkerTree};
use pypi_types::{ParsedUrlError, Yanked};
use pypi_types::{ParsedUrlError, Requirement, Yanked};
use uv_configuration::{Constraints, Overrides};
use uv_git::GitResolver;
use uv_normalize::{ExtraName, GroupName, PackageName};
@ -21,7 +22,7 @@ use crate::redirect::url_to_precise;
use crate::resolution::AnnotatedDist;
use crate::resolver::{Resolution, ResolutionPackage};
use crate::{
InMemoryIndex, Manifest, MetadataResponse, PythonRequirement, RequiresPython, ResolveError,
InMemoryIndex, MetadataResponse, PythonRequirement, RequiresPython, ResolveError,
VersionsResponse,
};
@ -35,6 +36,12 @@ pub struct ResolutionGraph {
pub(crate) requires_python: Option<RequiresPython>,
/// Any diagnostics that were encountered while building the graph.
pub(crate) diagnostics: Vec<ResolutionDiagnostic>,
/// The requirements that were used to build the graph.
pub(crate) requirements: Vec<Requirement>,
/// The constraints that were used to build the graph.
pub(crate) constraints: Constraints,
/// The overrides that were used to build the graph.
pub(crate) overrides: Overrides,
}
type NodeKey<'a> = (
@ -46,9 +53,13 @@ type NodeKey<'a> = (
impl ResolutionGraph {
/// Create a new graph from the resolved PubGrub state.
#[allow(clippy::too_many_arguments)]
pub(crate) fn from_state(
index: &InMemoryIndex,
requirements: &[Requirement],
constraints: &Constraints,
overrides: &Overrides,
preferences: &Preferences,
index: &InMemoryIndex,
git: &GitResolver,
python: &PythonRequirement,
resolution: Resolution,
@ -274,6 +285,9 @@ impl ResolutionGraph {
petgraph,
requires_python,
diagnostics,
requirements: requirements.to_vec(),
constraints: constraints.clone(),
overrides: overrides.clone(),
})
}
@ -305,7 +319,7 @@ impl ResolutionGraph {
/// Return the marker tree specific to this resolution.
///
/// This accepts a manifest, in-memory-index and marker environment. All
/// This accepts an in-memory-index and marker environment, all
/// of which should be the same values given to the resolver that produced
/// this graph.
///
@ -325,7 +339,6 @@ impl ResolutionGraph {
/// to compute in some cases.)
pub fn marker_tree(
&self,
manifest: &Manifest,
index: &InMemoryIndex,
marker_env: &MarkerEnvironment,
) -> Result<MarkerTree, Box<ParsedUrlError>> {
@ -394,7 +407,10 @@ impl ResolutionGraph {
dist.version_id()
)
};
for req in manifest.apply(archive.metadata.requires_dist.iter()) {
for req in self
.constraints
.apply(self.overrides.apply(archive.metadata.requires_dist.iter()))
{
let Some(ref marker_tree) = req.marker else {
continue;
};
@ -403,7 +419,10 @@ impl ResolutionGraph {
}
// Ensure that we consider markers from direct dependencies.
for direct_req in manifest.apply(manifest.requirements.iter()) {
for direct_req in self
.constraints
.apply(self.overrides.apply(self.requirements.iter()))
{
let Some(ref marker_tree) = direct_req.marker else {
continue;
};

5
crates/uv-resolver/src/resolver/mod.rs
View file

@ -573,8 +573,11 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
combined.union(resolution);
}
ResolutionGraph::from_state(
&self.index,
&self.requirements,
&self.constraints,
&self.overrides,
&self.preferences,
&self.index,
&self.git,
&self.python_requirement,
combined,

277
crates/uv/src/commands/pip/compile.rs
View file

@ -1,5 +1,4 @@
use std::env;
use std::fmt::Write;
use std::io::stdout;
use std::ops::Deref;
use std::path::Path;
@ -10,33 +9,27 @@ use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::{
IndexLocations, SourceAnnotation, SourceAnnotations, UnresolvedRequirementSpecification,
Verbatim,
};
use distribution_types::{IndexLocations, UnresolvedRequirementSpecification, Verbatim};
use install_wheel_rs::linker::LinkMode;
use pypi_types::Requirement;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, IndexStrategy,
NoBinary, NoBuild, Overrides, PreviewMode, SetupPyStrategy, Upgrade,
BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, IndexStrategy, NoBinary,
NoBuild, PreviewMode, Reinstall, SetupPyStrategy, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
use uv_fs::Simplified;
use uv_git::GitResolver;
use uv_normalize::PackageName;
use uv_requirements::{
upgrade::read_requirements_txt, LookaheadResolver, NamedRequirementsResolver,
RequirementsSource, RequirementsSpecification, SourceTreeResolver,
upgrade::read_requirements_txt, RequirementsSource, RequirementsSpecification,
};
use uv_resolver::{
AnnotationStyle, DependencyMode, DisplayResolutionGraph, ExcludeNewer, Exclusions, FlatIndex,
InMemoryIndex, Manifest, OptionsBuilder, PreReleaseMode, PythonRequirement, ResolutionMode,
Resolver,
AnnotationStyle, DependencyMode, DisplayResolutionGraph, ExcludeNewer, FlatIndex,
InMemoryIndex, OptionsBuilder, PreReleaseMode, PythonRequirement, ResolutionMode,
};
use uv_toolchain::{
EnvironmentPreference, PythonEnvironment, PythonVersion, Toolchain, ToolchainPreference,
@ -46,8 +39,7 @@ use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;
use crate::commands::pip::{operations, resolution_environment};
use crate::commands::reporters::ResolverReporter;
use crate::commands::{elapsed, ExitStatus};
use crate::commands::ExitStatus;
use crate::printer::Printer;
/// Resolve a set of requirements into a set of pinned versions.
@ -97,8 +89,6 @@ pub(crate) async fn pip_compile(
cache: Cache,
printer: Printer,
) -> Result<ExitStatus> {
let start = std::time::Instant::now();
// If the user requests `extras` but does not provide a valid source (e.g., a `pyproject.toml`),
// return an error.
if !extras.is_empty() && !requirements.iter().any(RequirementsSource::allows_extras) {
@ -114,7 +104,7 @@ pub(crate) async fn pip_compile(
// Read all requirements from the provided sources.
let RequirementsSpecification {
mut project,
project,
requirements,
constraints,
overrides,
@ -134,6 +124,16 @@ pub(crate) async fn pip_compile(
)
.await?;
let overrides: Vec<UnresolvedRequirementSpecification> = overrides
.iter()
.cloned()
.chain(
overrides_from_workspace
.into_iter()
.map(UnresolvedRequirementSpecification::from),
)
.collect();
// If all the metadata could be statically resolved, validate that every extra was used. If we
// need to resolve metadata via PEP 517, we don't know which extras are used until much later.
if source_trees.is_empty() {
@ -210,12 +210,10 @@ pub(crate) async fn pip_compile(
InMemoryIndexRef::Borrowed(&source_index)
};
// Determine the Python requirement, based on the interpreter and the requested version.
let python_requirement = if let Some(python_version) = python_version.as_ref() {
PythonRequirement::from_python_version(&interpreter, python_version)
} else {
PythonRequirement::from_interpreter(&interpreter)
};
// Determine the Python requirement, if the user requested a specific version.
let python_requirement = python_version
.as_ref()
.map(|python_version| PythonRequirement::from_python_version(&interpreter, python_version));
// Determine the environment for the resolution.
let (tags, markers) = resolution_environment(python_version, python_platform, &interpreter)?;
@ -227,6 +225,9 @@ pub(crate) async fn pip_compile(
HashStrategy::None
};
// Ignore development dependencies.
let dev = Vec::default();
// Incorporate any index locations from the provided sources.
let index_locations =
index_locations.combine(index_url, extra_index_urls, find_links, no_index);
@ -293,173 +294,6 @@ pub(crate) async fn pip_compile(
preview,
);
// Resolve the requirements from the provided sources.
let requirements = {
// Convert from unnamed to named requirements.
let mut requirements = NamedRequirementsResolver::new(
requirements,
&hasher,
&top_level_index,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads, preview),
)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;
// Resolve any source trees into requirements.
if !source_trees.is_empty() {
let resolutions = SourceTreeResolver::new(
source_trees,
&extras,
&hasher,
&top_level_index,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads, preview),
)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;
// If we resolved a single project, use it for the project name.
project = project.or_else(|| {
if let [resolution] = &resolutions[..] {
Some(resolution.project.clone())
} else {
None
}
});
// If any of the extras were unused, surface a warning.
if let ExtrasSpecification::Some(extras) = extras {
let mut unused_extras = extras
.iter()
.filter(|extra| {
!resolutions
.iter()
.any(|resolution| resolution.extras.contains(extra))
})
.collect::<Vec<_>>();
if !unused_extras.is_empty() {
unused_extras.sort_unstable();
unused_extras.dedup();
let s = if unused_extras.len() == 1 { "" } else { "s" };
return Err(anyhow!(
"Requested extra{s} not found: {}",
unused_extras.iter().join(", ")
));
}
}
// Extend the requirements with the resolved source trees.
requirements.extend(
resolutions
.into_iter()
.flat_map(|resolution| resolution.requirements),
);
}
requirements
};
// Merge workspace overrides.
let overrides: Vec<UnresolvedRequirementSpecification> = overrides
.iter()
.cloned()
.chain(
overrides_from_workspace
.into_iter()
.map(UnresolvedRequirementSpecification::from),
)
.collect();
// Resolve the overrides from the provided sources.
let overrides = NamedRequirementsResolver::new(
overrides,
&hasher,
&top_level_index,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads, preview),
)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;
// Generate a map from requirement to originating source file.
let mut sources = SourceAnnotations::default();
for requirement in requirements
.iter()
.filter(|requirement| requirement.evaluate_markers(Some(&markers), &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Requirement(origin.clone()),
);
}
}
for requirement in constraints
.iter()
.filter(|requirement| requirement.evaluate_markers(Some(&markers), &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Constraint(origin.clone()),
);
}
}
for requirement in overrides
.iter()
.filter(|requirement| requirement.evaluate_markers(Some(&markers), &[]))
{
if let Some(origin) = &requirement.origin {
sources.add(
&requirement.name,
SourceAnnotation::Override(origin.clone()),
);
}
}
// Collect constraints and overrides.
let constraints = Constraints::from_requirements(constraints);
let overrides = Overrides::from_requirements(overrides);
// Ignore development dependencies.
let dev = Vec::default();
// Determine any lookahead requirements.
let lookaheads = match dependency_mode {
DependencyMode::Transitive => {
LookaheadResolver::new(
&requirements,
&constraints,
&overrides,
&dev,
&hasher,
&top_level_index,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads, preview),
)
.with_reporter(ResolverReporter::from(printer))
.resolve(Some(&markers))
.await?
}
DependencyMode::Direct => Vec::new(),
};
// Create a manifest of the requirements.
let manifest = Manifest::new(
requirements,
constraints,
overrides,
dev,
preferences,
project,
// Do not consider any installed packages during resolution.
Exclusions::All,
lookaheads,
);
let options = OptionsBuilder::new()
.resolution_mode(resolution_mode)
.prerelease_mode(prerelease_mode)
@ -468,43 +302,44 @@ pub(crate) async fn pip_compile(
.index_strategy(index_strategy)
.build();
// Resolve the dependencies.
let resolver = Resolver::new(
manifest.clone(),
options,
&python_requirement,
Some(&markers),
// Resolve the requirements.
let resolution = match operations::resolve(
requirements,
constraints,
overrides,
dev,
source_trees,
project,
&extras,
preferences,
EmptyInstalledPackages,
&hasher,
&Reinstall::None,
&upgrade,
&interpreter,
Some(&tags),
Some(&markers),
python_requirement,
&client,
&flat_index,
&top_level_index,
&hasher,
&build_dispatch,
EmptyInstalledPackages,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads, preview),
)?
.with_reporter(ResolverReporter::from(printer));
let resolution = match resolver.resolve().await {
Err(uv_resolver::ResolveError::NoSolution(err)) => {
concurrency,
options,
printer,
preview,
)
.await
{
Ok(resolution) => resolution,
Err(operations::Error::Resolve(uv_resolver::ResolveError::NoSolution(err))) => {
let report = miette::Report::msg(format!("{err}"))
.context("No solution found when resolving dependencies:");
eprint!("{report:?}");
return Ok(ExitStatus::Failure);
}
result => result,
}?;
let s = if resolution.len() == 1 { "" } else { "s" };
writeln!(
printer.stderr(),
"{}",
format!(
"Resolved {} in {}",
format!("{} package{}", resolution.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
Err(err) => return Err(err.into()),
};
// Write the resolved dependencies to the output channel.
let mut writer = OutputWriter::new(!quiet || output_file.is_none(), output_file)?;
@ -531,7 +366,7 @@ pub(crate) async fn pip_compile(
}
if include_marker_expression {
let relevant_markers = resolution.marker_tree(&manifest, &top_level_index, &markers)?;
let relevant_markers = resolution.marker_tree(&top_level_index, &markers)?;
writeln!(
writer,
"{}",
@ -602,13 +437,13 @@ pub(crate) async fn pip_compile(
"{}",
DisplayResolutionGraph::new(
&resolution,
Some(&markers),
&no_emit_packages,
generate_hashes,
include_extras,
include_annotations,
include_index_annotation,
annotation_style,
sources,
)
)?;

13
crates/uv/src/commands/pip/operations.rs
View file

@ -35,7 +35,7 @@ use uv_requirements::{
};
use uv_resolver::{
DependencyMode, Exclusions, FlatIndex, InMemoryIndex, Manifest, Options, Preference,
PythonRequirement, RequiresPython, ResolutionGraph, Resolver,
PythonRequirement, ResolutionGraph, Resolver,
};
use uv_toolchain::{Interpreter, PythonEnvironment};
use uv_types::{HashStrategy, InFlight, InstalledPackagesProvider};
@ -90,7 +90,7 @@ pub(crate) async fn resolve<InstalledPackages: InstalledPackagesProvider>(
interpreter: &Interpreter,
tags: Option<&Tags>,
markers: Option<&MarkerEnvironment>,
requires_python: Option<&RequiresPython>,
python_requirement: Option<PythonRequirement>,
client: &RegistryClient,
flat_index: &FlatIndex,
index: &InMemoryIndex,
@ -184,11 +184,10 @@ pub(crate) async fn resolve<InstalledPackages: InstalledPackagesProvider>(
// Collect constraints and overrides.
let constraints = Constraints::from_requirements(constraints);
let overrides = Overrides::from_requirements(overrides);
let python_requirement = if let Some(requires_python) = requires_python {
PythonRequirement::from_requires_python(interpreter, requires_python)
} else {
PythonRequirement::from_interpreter(interpreter)
};
// Determine the Python requirement, defaulting to that of the interpreter.
let python_requirement =
python_requirement.unwrap_or_else(|| PythonRequirement::from_interpreter(interpreter));
// Determine any lookahead requirements.
let lookaheads = match options.dependency_mode {

8
crates/uv/src/commands/project/lock.rs
View file

@ -10,7 +10,9 @@ use uv_dispatch::BuildDispatch;
use uv_distribution::{Workspace, DEV_DEPENDENCIES};
use uv_git::GitResolver;
use uv_requirements::upgrade::{read_lockfile, LockedRequirements};
use uv_resolver::{FlatIndex, InMemoryIndex, Lock, OptionsBuilder, RequiresPython};
use uv_resolver::{
FlatIndex, InMemoryIndex, Lock, OptionsBuilder, PythonRequirement, RequiresPython,
};
use uv_toolchain::{Interpreter, ToolchainPreference, ToolchainRequest};
use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;
@ -139,6 +141,8 @@ pub(super) async fn do_lock(
default
};
let python_requirement = PythonRequirement::from_requires_python(interpreter, &requires_python);
// Initialize the registry client.
let client = RegistryClientBuilder::new(cache.clone())
.native_tls(native_tls)
@ -219,7 +223,7 @@ pub(super) async fn do_lock(
interpreter,
None,
None,
Some(&requires_python),
Some(python_requirement),
&client,
&flat_index,
&index,