From 1f31350d46a8bf4959e37f36a69720932fe30352 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 28 Mar 2024 18:53:57 -0400 Subject: [PATCH] Recursively allow URL requirements for local dependencies (#2702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This is a trimmed-down version of https://github.com/astral-sh/uv/pull/2684 that only applies to local source trees for now, which enables workspace-like workflows (whereby local packages can depend on other local packages at arbitrary depth). Closes #2699. ## Test Plan Added new tests. Also cloned this MRE that was shared with me (https://github.com/timothyjlaurent/uv-poetry-monorepo-mre), and verified that it was installed without error: ``` ❯ cargo run pip install ./uv-poetry-monorepo-mre/app --no-cache Finished dev [unoptimized + debuginfo] target(s) in 0.15s Running `target/debug/uv pip install ./uv-poetry-monorepo-mre/app --no-cache` Resolved 4 packages in 1.28s Built app @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/app Built lib1 @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib1 Built lib2 @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib2 Downloaded 4 packages in 457ms Installed 4 packages in 2ms + app==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/app) + lib1==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib1) + lib2==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib2) + ruff==0.3.4 ``` --- crates/uv-requirements/src/lookahead.rs | 56 ++++++++++------ crates/uv/src/commands/pip_compile.rs | 28 +++++--- crates/uv/src/commands/pip_install.rs | 24 +++++-- crates/uv/tests/pip_compile.rs | 86 +++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 33 deletions(-) diff --git a/crates/uv-requirements/src/lookahead.rs b/crates/uv-requirements/src/lookahead.rs index 835052804..e6a6f18b8 100644 --- a/crates/uv-requirements/src/lookahead.rs +++ b/crates/uv-requirements/src/lookahead.rs @@ -1,10 +1,12 @@ +use std::collections::VecDeque; use std::sync::Arc; -use anyhow::{Context, Result}; -use futures::{StreamExt, TryStreamExt}; +use anyhow::Result; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use distribution_types::{BuildableSource, Dist}; -use pep508_rs::{Requirement, VersionOrUrl}; +use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl}; use uv_client::RegistryClient; use uv_distribution::{Reporter, SourceDistCachedBuilder}; use uv_types::{BuildContext, RequestedRequirements}; @@ -20,16 +22,16 @@ use uv_types::{BuildContext, RequestedRequirements}; /// /// The lookahead resolver resolves requirements for local dependencies, so that the resolver can /// treat them as first-party dependencies for the purpose of analyzing their specifiers. -pub struct LookaheadResolver<'a> { +pub struct LookaheadResolver { /// The requirements for the project. - requirements: &'a [Requirement], + requirements: Vec, /// The reporter to use when building source distributions. reporter: Option>, } -impl<'a> LookaheadResolver<'a> { - /// Instantiate a new [`LookaheadResolver`] for a given set of `source_trees`. - pub fn new(requirements: &'a [Requirement]) -> Self { +impl LookaheadResolver { + /// Instantiate a new [`LookaheadResolver`] for a given set of requirements. + pub fn new(requirements: Vec) -> Self { Self { requirements, reporter: None, @@ -50,20 +52,37 @@ impl<'a> LookaheadResolver<'a> { pub async fn resolve( self, context: &T, + markers: &MarkerEnvironment, client: &RegistryClient, ) -> Result> { - let requirements: Vec<_> = futures::stream::iter(self.requirements.iter()) - .map(|requirement| async { self.lookahead(requirement, context, client).await }) - .buffered(50) - .try_collect() - .await?; - Ok(requirements.into_iter().flatten().collect()) + let mut queue = VecDeque::from(self.requirements.clone()); + let mut results = Vec::new(); + let mut futures = FuturesUnordered::new(); + + while !queue.is_empty() || !futures.is_empty() { + while let Some(requirement) = queue.pop_front() { + futures.push(self.lookahead(requirement, context, client)); + } + + while let Some(result) = futures.next().await { + if let Some(lookahead) = result? { + for requirement in lookahead.requirements() { + if requirement.evaluate_markers(markers, lookahead.extras()) { + queue.push_back(requirement.clone()); + } + } + results.push(lookahead); + } + } + } + + Ok(results) } /// Infer the package name for a given "unnamed" requirement. async fn lookahead( &self, - requirement: &Requirement, + requirement: Requirement, context: &T, client: &RegistryClient, ) -> Result> { @@ -73,7 +92,7 @@ impl<'a> LookaheadResolver<'a> { }; // Convert to a buildable distribution. - let dist = Dist::from_url(requirement.name.clone(), url.clone())?; + let dist = Dist::from_url(requirement.name, url.clone())?; // Only support source trees (and not, e.g., wheels). let Dist::Source(source_dist) = &dist else { @@ -92,12 +111,11 @@ impl<'a> LookaheadResolver<'a> { let metadata = builder .download_and_build_metadata(&BuildableSource::Dist(source_dist)) - .await - .context("Failed to build source distribution")?; + .await?; // Return the requirements from the metadata. Ok(Some(RequestedRequirements::new( - requirement.extras.clone(), + requirement.extras, metadata.requires_dist, ))) } diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 853d62a0f..320addcd8 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -277,14 +277,8 @@ pub(crate) async fn pip_compile( requirements }; - // Determine any lookahead requirements. - let lookaheads = LookaheadResolver::new(&requirements) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&build_dispatch, &client) - .await?; - // Build the editables and add their requirements - let editable_metadata = if editables.is_empty() { + let editables = if editables.is_empty() { Vec::new() } else { let start = std::time::Instant::now(); @@ -339,6 +333,24 @@ pub(crate) async fn pip_compile( editables }; + // Determine any lookahead requirements. + let lookaheads = LookaheadResolver::new( + requirements + .iter() + .filter(|requirement| requirement.evaluate_markers(&markers, &[])) + .chain(editables.iter().flat_map(|(editable, metadata)| { + metadata + .requires_dist + .iter() + .filter(|requirement| requirement.evaluate_markers(&markers, &editable.extras)) + })) + .cloned() + .collect(), + ) + .with_reporter(ResolverReporter::from(printer)) + .resolve(&build_dispatch, &markers, &client) + .await?; + // Create a manifest of the requirements. let manifest = Manifest::new( requirements, @@ -346,7 +358,7 @@ pub(crate) async fn pip_compile( overrides, preferences, project, - editable_metadata, + editables, // Do not consider any installed packages during compilation Exclusions::All, lookaheads, diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 6b7acb182..135933a0a 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -16,7 +16,7 @@ use distribution_types::{ use install_wheel_rs::linker::LinkMode; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; -use pypi_types::Yanked; +use pypi_types::{Metadata23, Yanked}; use requirements_txt::EditableRequirement; use uv_auth::{KeyringProvider, GLOBAL_AUTH_STORE}; use uv_cache::Cache; @@ -524,7 +524,7 @@ async fn resolve( .collect(); // Map the editables to their metadata. - let editables = editables + let editables: Vec<(LocalEditable, Metadata23)> = editables .iter() .map(|built_editable| { ( @@ -535,10 +535,22 @@ async fn resolve( .collect(); // Determine any lookahead requirements. - let lookaheads = LookaheadResolver::new(&requirements) - .with_reporter(ResolverReporter::from(printer)) - .resolve(build_dispatch, client) - .await?; + let lookaheads = LookaheadResolver::new( + requirements + .iter() + .filter(|requirement| requirement.evaluate_markers(markers, &[])) + .chain(editables.iter().flat_map(|(editable, metadata)| { + metadata + .requires_dist + .iter() + .filter(|requirement| requirement.evaluate_markers(markers, &editable.extras)) + })) + .cloned() + .collect(), + ) + .with_reporter(ResolverReporter::from(printer)) + .resolve(build_dispatch, markers, client) + .await?; // Create a manifest of the requirements. let manifest = Manifest::new( diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 7400594eb..1342b4785 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -6542,6 +6542,92 @@ fn pendulum_no_tzdata_on_windows() -> Result<()> { Ok(()) } +/// Allow URL dependencies recursively for local source trees. +#[test] +fn allow_recursive_url_local_path() -> Result<()> { + let context = TestContext::new("3.12"); + + // Create a standalone library. + let lib2 = context.temp_dir.child("lib2"); + lib2.create_dir_all()?; + let pyproject_toml = lib2.child("pyproject.toml"); + pyproject_toml.write_str( + r#"[project] +name = "lib2" +version = "0.0.0" +dependencies = [ + "idna" +] +requires-python = ">3.8" +"#, + )?; + + // Create a library that depends on the standalone library. + let lib1 = context.temp_dir.child("lib1"); + lib1.create_dir_all()?; + let pyproject_toml = lib1.child("pyproject.toml"); + pyproject_toml.write_str(&format!( + r#"[project] +name = "lib1" +version = "0.0.0" +dependencies = [ + "lib2 @ {}" +] +requires-python = ">3.8" +"#, + Url::from_directory_path(lib2.path()).unwrap().as_str(), + ))?; + + // Create an application that depends on the library. + let app = context.temp_dir.child("app"); + app.create_dir_all()?; + let pyproject_toml = app.child("pyproject.toml"); + pyproject_toml.write_str(&format!( + r#"[project] +name = "example" +version = "0.0.0" +dependencies = [ + "anyio", + "lib1 @ {}" +] +requires-python = ">3.8" +"#, + Url::from_directory_path(lib1.path()).unwrap().as_str(), + ))?; + + // Write to a requirements file. + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("./app")?; + + uv_snapshot!(context.filters(), context.compile() + .arg("requirements.in"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in + anyio==4.3.0 + # via example + example @ ./app + idna==3.6 + # via + # anyio + # lib2 + lib1 @ file://[TEMP_DIR]/lib1/ + # via example + lib2 @ file://[TEMP_DIR]/lib2/ + # via lib1 + sniffio==1.3.1 + # via anyio + + ----- stderr ----- + Resolved 6 packages in [TIME] + "### + ); + + Ok(()) +} + /// Allow pre-releases for dependencies of source path requirements. #[test] fn pre_release_path_requirement() -> Result<()> {