diff --git a/crates/distribution-types/src/buildable.rs b/crates/distribution-types/src/buildable.rs index 828c427c2..1b105d299 100644 --- a/crates/distribution-types/src/buildable.rs +++ b/crates/distribution-types/src/buildable.rs @@ -54,6 +54,17 @@ pub enum SourceUrl<'a> { Path(PathSourceUrl<'a>), } +impl<'a> SourceUrl<'a> { + /// Return the [`Url`] of the source. + pub fn url(&self) -> &Url { + match self { + Self::Direct(dist) => dist.url, + Self::Git(dist) => dist.url, + Self::Path(dist) => dist.url, + } + } +} + impl std::fmt::Display for SourceUrl<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/uv-requirements/src/lookahead.rs b/crates/uv-requirements/src/lookahead.rs index 1275352da..ad8e6effe 100644 --- a/crates/uv-requirements/src/lookahead.rs +++ b/crates/uv-requirements/src/lookahead.rs @@ -134,8 +134,9 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> { // Fetch the metadata for the distribution. let requires_dist = { - // If the metadata is already in the index, return it. - if let Some(metadata) = self.index.get_metadata(&dist.package_id()) { + let id = dist.package_id(); + if let Some(metadata) = self.index.get_metadata(&id) { + // If the metadata is already in the index, return it. metadata.requires_dist.clone() } else { // Run the PEP 517 build process to extract metadata from the source distribution. @@ -148,16 +149,17 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> { Dist::Source(source) => format!("Failed to download and build: {source}"), })?; + let requires_dist = metadata.requires_dist.clone(); + // Insert the metadata into the index. - self.index - .insert_metadata(dist.package_id(), metadata.clone()); + self.index.insert_metadata(id, metadata); // Insert the redirect into the index. if let Some(precise) = precise { self.index.insert_redirect(CanonicalUrl::new(url), precise); } - metadata.requires_dist + requires_dist } }; diff --git a/crates/uv-requirements/src/source_tree.rs b/crates/uv-requirements/src/source_tree.rs index 9560be192..79408d752 100644 --- a/crates/uv-requirements/src/source_tree.rs +++ b/crates/uv-requirements/src/source_tree.rs @@ -1,15 +1,17 @@ use std::borrow::Cow; +use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::Arc; use anyhow::{Context, Result}; use futures::{StreamExt, TryStreamExt}; use url::Url; -use distribution_types::{BuildableSource, PathSourceUrl, SourceUrl}; +use cache_key::CanonicalUrl; +use distribution_types::{BuildableSource, PackageId, PathSourceUrl, SourceUrl}; use pep508_rs::Requirement; use uv_client::RegistryClient; -use uv_distribution::{Reporter, SourceDistributionBuilder}; +use uv_distribution::{DistributionDatabase, Reporter}; +use uv_resolver::InMemoryIndex; use uv_types::BuildContext; use crate::ExtrasSpecification; @@ -18,45 +20,47 @@ use crate::ExtrasSpecification; /// /// Used, e.g., to determine the the input requirements when a user specifies a `pyproject.toml` /// file, which may require running PEP 517 build hooks to extract metadata. -pub struct SourceTreeResolver<'a> { +pub struct SourceTreeResolver<'a, Context: BuildContext + Send + Sync> { /// The requirements for the project. source_trees: Vec, /// The extras to include when resolving requirements. extras: &'a ExtrasSpecification<'a>, - /// The reporter to use when building source distributions. - reporter: Option>, + /// The in-memory index for resolving dependencies. + index: &'a InMemoryIndex, + /// The database for fetching and building distributions. + database: DistributionDatabase<'a, Context>, } -impl<'a> SourceTreeResolver<'a> { +impl<'a, Context: BuildContext + Send + Sync> SourceTreeResolver<'a, Context> { /// Instantiate a new [`SourceTreeResolver`] for a given set of `source_trees`. - pub fn new(source_trees: Vec, extras: &'a ExtrasSpecification<'a>) -> Self { + pub fn new( + source_trees: Vec, + extras: &'a ExtrasSpecification<'a>, + context: &'a Context, + client: &'a RegistryClient, + index: &'a InMemoryIndex, + ) -> Self { Self { source_trees, extras, - reporter: None, + index, + database: DistributionDatabase::new(client, context), } } /// Set the [`Reporter`] to use for this resolver. #[must_use] pub fn with_reporter(self, reporter: impl Reporter + 'static) -> Self { - let reporter: Arc = Arc::new(reporter); Self { - reporter: Some(reporter), + database: self.database.with_reporter(reporter), ..self } } /// Resolve the requirements from the provided source trees. - pub async fn resolve( - self, - context: &T, - client: &RegistryClient, - ) -> Result> { + pub async fn resolve(self) -> Result> { let requirements: Vec<_> = futures::stream::iter(self.source_trees.iter()) - .map(|source_tree| async { - self.resolve_source_tree(source_tree, context, client).await - }) + .map(|source_tree| async { self.resolve_source_tree(source_tree).await }) .buffered(50) .try_collect() .await?; @@ -64,12 +68,7 @@ impl<'a> SourceTreeResolver<'a> { } /// Infer the package name for a given "unnamed" requirement. - async fn resolve_source_tree( - &self, - source_tree: &Path, - context: &T, - client: &RegistryClient, - ) -> Result> { + async fn resolve_source_tree(&self, source_tree: &Path) -> Result> { // Convert to a buildable source. let path = fs_err::canonicalize(source_tree).with_context(|| { format!( @@ -80,20 +79,34 @@ impl<'a> SourceTreeResolver<'a> { let Ok(url) = Url::from_directory_path(&path) else { return Err(anyhow::anyhow!("Failed to convert path to URL")); }; - let source = BuildableSource::Url(SourceUrl::Path(PathSourceUrl { + let source = SourceUrl::Path(PathSourceUrl { url: &url, path: Cow::Owned(path), - })); + }); - // Run the PEP 517 build process to extract metadata from the source distribution. - let builder = if let Some(reporter) = self.reporter.clone() { - SourceDistributionBuilder::new(client, context).with_reporter(reporter) - } else { - SourceDistributionBuilder::new(client, context) + // Fetch the metadata for the distribution. + let metadata = { + let id = PackageId::from_url(source.url()); + if let Some(metadata) = self.index.get_metadata(&id) { + // If the metadata is already in the index, return it. + metadata.deref().clone() + } else { + // Run the PEP 517 build process to extract metadata from the source distribution. + let source = BuildableSource::Url(source); + let (metadata, precise) = self.database.build_wheel_metadata(&source).await?; + + // Insert the metadata into the index. + self.index.insert_metadata(id, metadata.clone()); + + // Insert the redirect into the index. + if let Some(precise) = precise { + self.index.insert_redirect(CanonicalUrl::new(&url), precise); + } + + metadata + } }; - let metadata = builder.download_and_build_metadata(&source).await?; - // Determine the appropriate requirements to return based on the extras. This involves // evaluating the `extras` expression in any markers, but preserving the remaining marker // conditions. diff --git a/crates/uv-requirements/src/unnamed.rs b/crates/uv-requirements/src/unnamed.rs index f4d237b2d..d80e218a2 100644 --- a/crates/uv-requirements/src/unnamed.rs +++ b/crates/uv-requirements/src/unnamed.rs @@ -1,72 +1,76 @@ use std::borrow::Cow; use std::path::Path; use std::str::FromStr; -use std::sync::Arc; -use anyhow::{Context, Result}; +use anyhow::Result; use configparser::ini::Ini; use futures::{StreamExt, TryStreamExt}; use serde::Deserialize; use tracing::debug; +use cache_key::CanonicalUrl; use distribution_filename::{SourceDistFilename, WheelFilename}; use distribution_types::{ - BuildableSource, DirectSourceUrl, GitSourceUrl, PathSourceUrl, RemoteSource, SourceUrl, + BuildableSource, DirectSourceUrl, GitSourceUrl, PackageId, PathSourceUrl, RemoteSource, + SourceUrl, }; use pep508_rs::{ Requirement, RequirementsTxtRequirement, Scheme, UnnamedRequirement, VersionOrUrl, }; use pypi_types::Metadata10; use uv_client::RegistryClient; -use uv_distribution::{Reporter, SourceDistributionBuilder}; +use uv_distribution::{DistributionDatabase, Reporter}; use uv_normalize::PackageName; +use uv_resolver::InMemoryIndex; use uv_types::BuildContext; /// Like [`RequirementsSpecification`], but with concrete names for all requirements. -pub struct NamedRequirementsResolver { +pub struct NamedRequirementsResolver<'a, Context: BuildContext + Send + Sync> { /// The requirements for the project. requirements: Vec, - /// The reporter to use when building source distributions. - reporter: Option>, + /// The in-memory index for resolving dependencies. + index: &'a InMemoryIndex, + /// The database for fetching and building distributions. + database: DistributionDatabase<'a, Context>, } -impl NamedRequirementsResolver { +impl<'a, Context: BuildContext + Send + Sync> NamedRequirementsResolver<'a, Context> { /// Instantiate a new [`NamedRequirementsResolver`] for a given set of requirements. - pub fn new(requirements: Vec) -> Self { + pub fn new( + requirements: Vec, + context: &'a Context, + client: &'a RegistryClient, + index: &'a InMemoryIndex, + ) -> Self { Self { requirements, - reporter: None, + index, + database: DistributionDatabase::new(client, context), } } /// Set the [`Reporter`] to use for this resolver. #[must_use] pub fn with_reporter(self, reporter: impl Reporter + 'static) -> Self { - let reporter: Arc = Arc::new(reporter); Self { - reporter: Some(reporter), + database: self.database.with_reporter(reporter), ..self } } /// Resolve any unnamed requirements in the specification. - pub async fn resolve( - self, - context: &T, - client: &RegistryClient, - ) -> Result> { - futures::stream::iter(self.requirements) + pub async fn resolve(self) -> Result> { + let Self { + requirements, + index, + database, + } = self; + futures::stream::iter(requirements) .map(|requirement| async { match requirement { RequirementsTxtRequirement::Pep508(requirement) => Ok(requirement), RequirementsTxtRequirement::Unnamed(requirement) => { - Self::resolve_requirement( - requirement, - context, - client, - self.reporter.clone(), - ) - .await + Self::resolve_requirement(requirement, index, &database).await } } }) @@ -76,11 +80,10 @@ impl NamedRequirementsResolver { } /// Infer the package name for a given "unnamed" requirement. - async fn resolve_requirement( + async fn resolve_requirement( requirement: UnnamedRequirement, - context: &T, - client: &RegistryClient, - reporter: Option>, + index: &InMemoryIndex, + database: &DistributionDatabase<'a, Context>, ) -> Result { // If the requirement is a wheel, extract the package name from the wheel filename. // @@ -231,20 +234,33 @@ impl NamedRequirementsResolver { } }; - // Run the PEP 517 build process to extract metadata from the source distribution. - let builder = if let Some(reporter) = reporter { - SourceDistributionBuilder::new(client, context).with_reporter(reporter) - } else { - SourceDistributionBuilder::new(client, context) + // Fetch the metadata for the distribution. + let name = { + let id = PackageId::from_url(source.url()); + if let Some(metadata) = index.get_metadata(&id) { + // If the metadata is already in the index, return it. + metadata.name.clone() + } else { + // Run the PEP 517 build process to extract metadata from the source distribution. + let source = BuildableSource::Url(source); + let (metadata, precise) = database.build_wheel_metadata(&source).await?; + + let name = metadata.name.clone(); + + // Insert the metadata into the index. + index.insert_metadata(id, metadata); + + // Insert the redirect into the index. + if let Some(precise) = precise { + index.insert_redirect(CanonicalUrl::new(&requirement.url), precise); + } + + name + } }; - let metadata = builder - .download_and_build_metadata(&BuildableSource::Url(source)) - .await - .context("Failed to build source distribution")?; - Ok(Requirement { - name: metadata.name, + name, extras: requirement.extras, version_or_url: Some(VersionOrUrl::Url(requirement.url)), marker: requirement.marker, diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 4c2b793c5..28c28978c 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -263,18 +263,29 @@ pub(crate) async fn pip_compile( // Resolve the requirements from the provided sources. let requirements = { // Convert from unnamed to named requirements. - let mut requirements = NamedRequirementsResolver::new(requirements) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&build_dispatch, &client) - .await?; + let mut requirements = NamedRequirementsResolver::new( + requirements, + &build_dispatch, + &client, + &top_level_index, + ) + .with_reporter(ResolverReporter::from(printer)) + .resolve() + .await?; // Resolve any source trees into requirements. if !source_trees.is_empty() { requirements.extend( - SourceTreeResolver::new(source_trees, &extras) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&build_dispatch, &client) - .await?, + SourceTreeResolver::new( + source_trees, + &extras, + &build_dispatch, + &client, + &top_level_index, + ) + .with_reporter(ResolverReporter::from(printer)) + .resolve() + .await?, ); } diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 3479a3bc0..e54d64ff0 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -244,17 +244,18 @@ pub(crate) async fn pip_install( // Resolve the requirements from the provided sources. let requirements = { // Convert from unnamed to named requirements. - let mut requirements = NamedRequirementsResolver::new(requirements) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&resolve_dispatch, &client) - .await?; + let mut requirements = + NamedRequirementsResolver::new(requirements, &resolve_dispatch, &client, &index) + .with_reporter(ResolverReporter::from(printer)) + .resolve() + .await?; // Resolve any source trees into requirements. if !source_trees.is_empty() { requirements.extend( - SourceTreeResolver::new(source_trees, extras) + SourceTreeResolver::new(source_trees, extras, &resolve_dispatch, &client, &index) .with_reporter(ResolverReporter::from(printer)) - .resolve(&resolve_dispatch, &client) + .resolve() .await?, ); } diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index e994a213f..2ad3942c2 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -194,18 +194,25 @@ pub(crate) async fn pip_sync( // Convert from unnamed to named requirements. let requirements = { // Convert from unnamed to named requirements. - let mut requirements = NamedRequirementsResolver::new(requirements) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&build_dispatch, &client) - .await?; + let mut requirements = + NamedRequirementsResolver::new(requirements, &build_dispatch, &client, &index) + .with_reporter(ResolverReporter::from(printer)) + .resolve() + .await?; // Resolve any source trees into requirements. if !source_trees.is_empty() { requirements.extend( - SourceTreeResolver::new(source_trees, &ExtrasSpecification::None) - .with_reporter(ResolverReporter::from(printer)) - .resolve(&build_dispatch, &client) - .await?, + SourceTreeResolver::new( + source_trees, + &ExtrasSpecification::None, + &build_dispatch, + &client, + &index, + ) + .with_reporter(ResolverReporter::from(printer)) + .resolve() + .await?, ); } diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index eb31bf0fd..281c8b7f5 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -1354,6 +1354,33 @@ fn compile_git_concurrent_access() -> Result<()> { Ok(()) } +/// Resolve two packages from a `requirements.in` file with the same Git HTTPS dependency. +#[test] +#[cfg(feature = "git")] +fn compile_git_unnamed_concurrent_access() -> Result<()> { + let context = TestContext::new("3.12"); + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in + .write_str("git+https://github.com/pypa/sample-namespace-packages.git@df7530eeb8fa0cb7dbb8ecb28363e8e36bfa2f45#subdirectory=pkg_resources/pkg_a\ngit+https://github.com/pypa/sample-namespace-packages.git@df7530eeb8fa0cb7dbb8ecb28363e8e36bfa2f45#subdirectory=pkg_resources/pkg_b")?; + + uv_snapshot!(context.compile() + .arg("requirements.in"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in + example-pkg-a @ git+https://github.com/pypa/sample-namespace-packages.git@df7530eeb8fa0cb7dbb8ecb28363e8e36bfa2f45#subdirectory=pkg_resources/pkg_a + example-pkg-b @ git+https://github.com/pypa/sample-namespace-packages.git@df7530eeb8fa0cb7dbb8ecb28363e8e36bfa2f45#subdirectory=pkg_resources/pkg_b + + ----- stderr ----- + Resolved 2 packages in [TIME] + "### + ); + + Ok(()) +} + /// Resolve a Git dependency with a declared name that differs from the true name of the package. #[test] #[cfg(feature = "git")]