mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-11-02 04:48:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Support recursive extras with marker in pip compile -r pyproject.toml (#9535)

Browse source 
## Summary

Closes https://github.com/astral-sh/uv/issues/9530.
This commit is contained in:
Charlie Marsh 2024-11-29 22:40:22 -05:00 • committed by GitHub
parent 891e02d586
commit 69811837e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 209 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

83
crates/uv-requirements/src/source_tree.rs
View file

@ -1,10 +1,13 @@
use std::borrow::Cow;
use std::collections::VecDeque;
use std::path::Path;
use std::slice;
use std::sync::Arc;
use anyhow::{Context, Result};
use futures::stream::FuturesOrdered;
use futures::TryStreamExt;
use rustc_hash::FxHashSet;
use url::Url;
use uv_configuration::ExtrasSpecification;
@ -14,7 +17,7 @@ use uv_distribution_types::{
};
use uv_fs::Simplified;
use uv_normalize::{ExtraName, PackageName};
use uv_pep508::RequirementOrigin;
use uv_pep508::{MarkerTree, RequirementOrigin};
use uv_pypi_types::Requirement;
use uv_resolver::{InMemoryIndex, MetadataResponse};
use uv_types::{BuildContext, HashStrategy};
@ -89,16 +92,13 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
let origin = RequirementOrigin::Project(path.to_path_buf(), metadata.name.clone());
// Determine the extras to include when resolving the requirements.
let extras: Vec<_> = self
let extras = self
.extras
.extra_names(metadata.provides_extras.iter())
.cloned()
.collect();
.collect::<Vec<_>>();
// Determine the appropriate requirements to return based on the extras. This involves
// evaluating the `extras` expression in any markers, but preserving the remaining marker
// conditions.
let mut requirements: Vec<Requirement> = metadata
let dependencies = metadata
.requires_dist
.into_iter()
.map(|requirement| Requirement {
@ -106,30 +106,61 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
marker: requirement.marker.simplify_extras(&extras),
..requirement
})
.collect::<Vec<_>>();
// Transitively process all extras that are recursively included, starting with the current
// extra.
let mut requirements = dependencies.clone();
let mut seen = FxHashSet::<(ExtraName, MarkerTree)>::default();
let mut queue: VecDeque<_> = requirements
.iter()
.filter(|req| req.name == metadata.name)
.flat_map(|req| {
req.extras
.iter()
.cloned()
.map(|extra| (extra, req.marker.clone().simplify_extras(&extras)))
})
.collect();
while let Some((extra, marker)) = queue.pop_front() {
if !seen.insert((extra.clone(), marker.clone())) {
continue;
}
// Resolve any recursive extras.
loop {
// Find the first recursive requirement.
// TODO(charlie): Respect markers on recursive extras.
let Some(index) = requirements.iter().position(|requirement| {
requirement.name == metadata.name && requirement.marker.is_true()
}) else {
break;
};
// Remove the requirement that points to us.
let recursive = requirements.remove(index);
// Re-simplify the requirements.
for requirement in &mut requirements {
requirement.marker = requirement
.marker
.clone()
.simplify_extras(&recursive.extras);
// Find the requirements for the extra.
for requirement in &dependencies {
if requirement.marker.top_level_extra_name().as_ref() == Some(&extra) {
let requirement = {
let mut marker = marker.clone();
marker.and(requirement.marker.clone());
Requirement {
name: requirement.name.clone(),
extras: requirement.extras.clone(),
source: requirement.source.clone(),
origin: requirement.origin.clone(),
marker: marker.simplify_extras(slice::from_ref(&extra)),
}
};
if requirement.name == metadata.name {
// Add each transitively included extra.
queue.extend(
requirement
.extras
.iter()
.cloned()
.map(|extra| (extra, requirement.marker.clone())),
);
} else {
// Add the requirements for that extra.
requirements.push(requirement);
}
}
}
}
// Drop all the self-requirements now that we flattened them out.
requirements.retain(|req| req.name != metadata.name);
let project = metadata.name;
let extras = metadata.provides_extras;

50
crates/uv-resolver/src/resolver/mod.rs
View file

@ -7,7 +7,7 @@ use std::fmt::{Display, Formatter, Write};
use std::ops::Bound;
use std::sync::Arc;
use std::time::Instant;
use std::{iter, thread};
use std::{iter, slice, thread};
use dashmap::DashMap;
use either::Either;
@ -1262,11 +1262,9 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
let dependencies = match &**package {
PubGrubPackageInner::Root(_) => {
let no_dev_deps = BTreeMap::default();
let no_provides_extras = [];
let requirements = self.flatten_requirements(
&self.requirements,
&no_dev_deps,
&no_provides_extras,
None,
None,
None,
@ -1454,7 +1452,6 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
let requirements = self.flatten_requirements(
&metadata.requires_dist,
&metadata.dependency_groups,
&metadata.provides_extras,
extra.as_ref(),
dev.as_ref(),
Some(name),
@ -1579,7 +1576,6 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
&'a self,
dependencies: &'a [Requirement],
dev_dependencies: &'a BTreeMap<GroupName, Vec<Requirement>>,
extras: &'a [ExtraName],
extra: Option<&'a ExtraName>,
dev: Option<&'a GroupName>,
name: Option<&PackageName>,
@ -1622,7 +1618,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
req.extras
.iter()
.cloned()
.map(|extra| (extra, req.marker.clone().simplify_extras(extras)))
.map(|extra| (extra, req.marker.clone()))
})
.collect();
while let Some((extra, marker)) = queue.pop_front() {
@ -1632,37 +1628,35 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
for requirement in
self.requirements_for_extra(dependencies, Some(&extra), env, python_requirement)
{
let requirement = if marker.is_true() {
requirement
} else {
match requirement {
Cow::Owned(mut requirement) => {
requirement.marker.and(marker.clone());
Cow::Owned(requirement)
}
Cow::Borrowed(requirement) => {
let mut marker = marker.clone();
marker.and(requirement.marker.clone());
Cow::Owned(Requirement {
name: requirement.name.clone(),
extras: requirement.extras.clone(),
source: requirement.source.clone(),
origin: requirement.origin.clone(),
marker,
})
let requirement = match requirement {
Cow::Owned(mut requirement) => {
requirement.marker.and(marker.clone());
requirement
}
Cow::Borrowed(requirement) => {
let mut marker = marker.clone();
marker.and(requirement.marker.clone());
Requirement {
name: requirement.name.clone(),
extras: requirement.extras.clone(),
source: requirement.source.clone(),
origin: requirement.origin.clone(),
marker: marker.simplify_extras(slice::from_ref(&extra)),
}
}
};
if name == Some(&requirement.name) {
// Add each transitively included extra.
queue.extend(
requirement.extras.iter().cloned().map(|extra| {
(extra, requirement.marker.clone().simplify_extras(extras))
}),
requirement
.extras
.iter()
.cloned()
.map(|extra| (extra, requirement.marker.clone())),
);
} else {
// Add the requirements for that extra.
requirements.push(requirement);
requirements.push(Cow::Owned(requirement));
}
}
}

130
crates/uv/tests/it/pip_compile.rs
View file

@ -9870,6 +9870,136 @@ dev = [
Ok(())
}
/// Resolve from a `pyproject.toml` file with a recursive extra, with a marker attached.
#[test]
fn compile_pyproject_toml_recursive_extra_marker() -> Result<()> {
let context = TestContext::new("3.12");
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(
r#"
[project]
name = "project"
version = "0.0.1"
dependencies = [
"anyio"
]
[project.optional-dependencies]
test = [
"iniconfig",
]
dev = [
"project[test] ; sys_platform == 'darwin'",
]
"#,
)?;
uv_snapshot!(context.filters(), context.pip_compile()
.arg("pyproject.toml")
.arg("--extra")
.arg("dev")
.arg("--universal"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --extra dev --universal
anyio==4.3.0
# via project (pyproject.toml)
idna==3.6
# via anyio
iniconfig==2.0.0 ; sys_platform == 'darwin'
# via project (pyproject.toml)
sniffio==1.3.1
# via anyio
----- stderr -----
Resolved 4 packages in [TIME]
"###
);
Ok(())
}
/// Resolve from a `pyproject.toml` file with multiple recursive extras.
#[test]
fn compile_pyproject_toml_deeply_recursive_extra() -> Result<()> {
let context = TestContext::new("3.12");
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(
r#"
[project]
name = "project"
version = "0.0.1"
dependencies = []
[project.optional-dependencies]
foo = ["iniconfig"]
bar = ["project[foo]"]
baz = ["project[bar]"]
bop = ["project[bar] ; sys_platform == 'darwin'"]
qux = ["project[bop] ; python_version == '3.12'"]
"#,
)?;
uv_snapshot!(context.filters(), context.pip_compile()
.arg("pyproject.toml")
.arg("--universal")
.arg("--extra")
.arg("qux"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra qux
iniconfig==2.0.0 ; python_full_version < '3.13' and sys_platform == 'darwin'
# via project (pyproject.toml)
----- stderr -----
Resolved 1 package in [TIME]
"###
);
uv_snapshot!(context.filters(), context.pip_compile()
.arg("pyproject.toml")
.arg("--universal")
.arg("--extra")
.arg("bop")
.arg("--extra")
.arg("bar"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra bop --extra bar
iniconfig==2.0.0
# via project (pyproject.toml)
----- stderr -----
Resolved 1 package in [TIME]
"###
);
uv_snapshot!(context.filters(), context.pip_compile()
.arg("pyproject.toml")
.arg("--universal")
.arg("--all-extras"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --all-extras
iniconfig==2.0.0
# via project (pyproject.toml)
----- stderr -----
Resolved 1 package in [TIME]
"###
);
Ok(())
}
/// The dependencies of a local editable dependency should be considered "direct" dependencies.
#[test]
fn editable_direct_dependency() -> Result<()> {