diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index 9e31b91e7..96156fa05 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -5,9 +5,7 @@ pub use prerelease_mode::PreReleaseMode; pub use resolution::ResolutionGraph; pub use resolution_mode::ResolutionMode; pub use resolution_options::ResolutionOptions; -pub use resolver::{ - BuildId, DefaultResolverProvider, Reporter as ResolverReporter, Resolver, ResolverProvider, -}; +pub use resolver::{BuildId, Reporter as ResolverReporter, Resolver, ResolverProvider}; mod candidate_selector; mod error; diff --git a/crates/puffin-resolver/src/resolver/allowed_urls.rs b/crates/puffin-resolver/src/resolver/allowed_urls.rs new file mode 100644 index 000000000..198f3eb41 --- /dev/null +++ b/crates/puffin-resolver/src/resolver/allowed_urls.rs @@ -0,0 +1,17 @@ +use rustc_hash::FxHashSet; +use url::Url; + +#[derive(Debug, Default)] +pub(crate) struct AllowedUrls(FxHashSet); + +impl AllowedUrls { + pub(crate) fn contains(&self, url: &Url) -> bool { + self.0.contains(&cache_key::CanonicalUrl::new(url)) + } +} + +impl<'a> FromIterator<&'a Url> for AllowedUrls { + fn from_iter>(iter: T) -> Self { + Self(iter.into_iter().map(cache_key::CanonicalUrl::new).collect()) + } +} diff --git a/crates/puffin-resolver/src/resolver/index.rs b/crates/puffin-resolver/src/resolver/index.rs new file mode 100644 index 000000000..5bb42d811 --- /dev/null +++ b/crates/puffin-resolver/src/resolver/index.rs @@ -0,0 +1,26 @@ +use url::Url; + +use distribution_types::{IndexUrl, PackageId}; +use pep440_rs::VersionSpecifiers; +use puffin_normalize::PackageName; +use puffin_traits::OnceMap; +use pypi_types::{BaseUrl, Metadata21}; + +use crate::version_map::VersionMap; + +/// In-memory index of package metadata. +#[derive(Default)] +pub(crate) struct Index { + /// A map from package name to the metadata for that package and the index where the metadata + /// came from. + pub(crate) packages: OnceMap, + + /// A map from package ID to metadata for that distribution. + pub(crate) distributions: OnceMap, + + /// A map from package ID to required Python version. + pub(crate) incompatibilities: OnceMap, + + /// A map from source URL to precise URL. + pub(crate) redirects: OnceMap, +} diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver/mod.rs similarity index 84% rename from crates/puffin-resolver/src/resolver.rs rename to crates/puffin-resolver/src/resolver/mod.rs index ed8d2e7a8..b47c33b8c 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver/mod.rs @@ -1,12 +1,10 @@ //! Given a set of requirements, find a set of compatible packages. -use std::future::Future; use std::sync::Arc; use anyhow::Result; -use chrono::{DateTime, Utc}; use futures::channel::mpsc::UnboundedReceiver; -use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; +use futures::{pin_mut, FutureExt, StreamExt}; use itertools::Itertools; use pubgrub::error::PubGrubError; use pubgrub::range::Range; @@ -19,17 +17,15 @@ use url::Url; use distribution_filename::WheelFilename; use distribution_types::{ - BuiltDist, Dist, DistributionMetadata, IndexUrl, LocalEditable, Name, PackageId, SourceDist, - VersionOrUrl, + BuiltDist, Dist, DistributionMetadata, IndexUrl, LocalEditable, Name, SourceDist, VersionOrUrl, }; -use pep440_rs::VersionSpecifiers; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; use puffin_client::RegistryClient; -use puffin_distribution::{DistributionDatabase, DistributionDatabaseError}; +use puffin_distribution::DistributionDatabase; use puffin_interpreter::Interpreter; use puffin_normalize::PackageName; -use puffin_traits::{BuildContext, OnceMap}; +use puffin_traits::BuildContext; use pypi_types::{BaseUrl, Metadata21}; use crate::candidate_selector::CandidateSelector; @@ -43,107 +39,19 @@ use crate::pubgrub::{ }; use crate::python_requirement::PythonRequirement; use crate::resolution::ResolutionGraph; +use crate::resolver::allowed_urls::AllowedUrls; +use crate::resolver::index::Index; +use crate::resolver::provider::DefaultResolverProvider; +pub use crate::resolver::provider::ResolverProvider; +use crate::resolver::reporter::Facade; +pub use crate::resolver::reporter::{BuildId, Reporter}; use crate::version_map::VersionMap; -use crate::yanks::AllowedYanks; use crate::ResolutionOptions; -type VersionMapResponse = Result<(IndexUrl, BaseUrl, VersionMap), puffin_client::Error>; -type WheelMetadataResponse = Result<(Metadata21, Option), DistributionDatabaseError>; - -pub trait ResolverProvider: Send + Sync { - /// Get the version map for a package. - fn get_version_map<'io>( - &'io self, - package_name: &'io PackageName, - ) -> impl Future + Send + 'io; - - /// Get the metadata for a distribution. - /// - /// For a wheel, this is done by querying it's (remote) metadata, for a source dist we - /// (fetch and) build the source distribution and return the metadata from the built - /// distribution. - fn get_or_build_wheel_metadata<'io>( - &'io self, - dist: &'io Dist, - ) -> impl Future + Send + 'io; - - /// Set the [`Reporter`] to use for this installer. - #[must_use] - fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self; -} - -/// The main IO backend for the resolver, which does cached requests network requests using the -/// [`RegistryClient`] and [`DistributionDatabase`]. -pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> { - client: &'a RegistryClient, - fetcher: DistributionDatabase<'a, Context>, - tags: &'a Tags, - python_requirement: PythonRequirement<'a>, - exclude_newer: Option>, - allowed_yanks: AllowedYanks, -} - -impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Context> { - pub fn new( - client: &'a RegistryClient, - fetcher: DistributionDatabase<'a, Context>, - tags: &'a Tags, - python_requirement: PythonRequirement<'a>, - exclude_newer: Option>, - allowed_yanks: AllowedYanks, - ) -> Self { - Self { - client, - fetcher, - tags, - python_requirement, - exclude_newer, - allowed_yanks, - } - } -} - -impl<'a, Context: BuildContext + Send + Sync> ResolverProvider - for DefaultResolverProvider<'a, Context> -{ - fn get_version_map<'io>( - &'io self, - package_name: &'io PackageName, - ) -> impl Future + Send + 'io { - self.client - .simple(package_name) - .map_ok(move |(index, base, metadata)| { - ( - index, - base, - VersionMap::from_metadata( - metadata, - package_name, - self.tags, - &self.python_requirement, - &self.allowed_yanks, - self.exclude_newer.as_ref(), - ), - ) - }) - } - - fn get_or_build_wheel_metadata<'io>( - &'io self, - dist: &'io Dist, - ) -> impl Future + Send + 'io { - self.fetcher.get_or_build_wheel_metadata(dist) - } - - /// Set the [`puffin_distribution::Reporter`] to use for this installer. - #[must_use] - fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self { - Self { - fetcher: self.fetcher.with_reporter(reporter), - ..self - } - } -} +mod allowed_urls; +mod index; +mod provider; +mod reporter; pub struct Resolver<'a, Provider: ResolverProvider> { project: Option, @@ -373,7 +281,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { // Pick the next compatible version. let version = match decision { None => { - debug!("No compatible version found for: {}", next); + debug!("No compatible version found for: {next}"); let term_intersection = state .partial_solution @@ -909,51 +817,6 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { } } -pub type BuildId = usize; - -pub trait Reporter: Send + Sync { - /// Callback to invoke when a dependency is resolved. - fn on_progress(&self, name: &PackageName, version: VersionOrUrl); - - /// Callback to invoke when the resolution is complete. - fn on_complete(&self); - - /// Callback to invoke when a source distribution build is kicked off. - fn on_build_start(&self, dist: &SourceDist) -> usize; - - /// Callback to invoke when a source distribution build is complete. - fn on_build_complete(&self, dist: &SourceDist, id: usize); - - /// Callback to invoke when a repository checkout begins. - fn on_checkout_start(&self, url: &Url, rev: &str) -> usize; - - /// Callback to invoke when a repository checkout completes. - fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize); -} - -/// A facade for converting from [`Reporter`] to [`puffin_distribution::Reporter`]. -struct Facade { - reporter: Arc, -} - -impl puffin_distribution::Reporter for Facade { - fn on_build_start(&self, dist: &SourceDist) -> usize { - self.reporter.on_build_start(dist) - } - - fn on_build_complete(&self, dist: &SourceDist, id: usize) { - self.reporter.on_build_complete(dist, id); - } - - fn on_checkout_start(&self, url: &Url, rev: &str) -> usize { - self.reporter.on_checkout_start(url, rev) - } - - fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize) { - self.reporter.on_checkout_complete(url, rev, index); - } -} - /// Fetch the metadata for an item #[derive(Debug)] #[allow(clippy::large_enum_variant)] @@ -975,38 +838,6 @@ enum Response { Dist(Dist, Metadata21, Option), } -/// In-memory index of package metadata. -#[derive(Default)] -pub(crate) struct Index { - /// A map from package name to the metadata for that package and the index where the metadata - /// came from. - pub(crate) packages: OnceMap, - - /// A map from package ID to metadata for that distribution. - pub(crate) distributions: OnceMap, - - /// A map from package ID to required Python version. - pub(crate) incompatibilities: OnceMap, - - /// A map from source URL to precise URL. - pub(crate) redirects: OnceMap, -} - -#[derive(Debug, Default)] -struct AllowedUrls(FxHashSet); - -impl AllowedUrls { - fn contains(&self, url: &Url) -> bool { - self.0.contains(&cache_key::CanonicalUrl::new(url)) - } -} - -impl<'a> FromIterator<&'a Url> for AllowedUrls { - fn from_iter>(iter: T) -> Self { - Self(iter.into_iter().map(cache_key::CanonicalUrl::new).collect()) - } -} - /// An enum used by [`DependencyProvider`] that holds information about package dependencies. /// For each [Package] there is a set of versions allowed as a dependency. #[derive(Clone)] diff --git a/crates/puffin-resolver/src/resolver/provider.rs b/crates/puffin-resolver/src/resolver/provider.rs new file mode 100644 index 000000000..86f76ae16 --- /dev/null +++ b/crates/puffin-resolver/src/resolver/provider.rs @@ -0,0 +1,116 @@ +use std::future::Future; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use futures::TryFutureExt; +use url::Url; + +use distribution_types::{Dist, IndexUrl}; +use platform_tags::Tags; +use puffin_client::RegistryClient; +use puffin_distribution::{DistributionDatabase, DistributionDatabaseError}; +use puffin_normalize::PackageName; +use puffin_traits::BuildContext; +use pypi_types::{BaseUrl, Metadata21}; + +use crate::python_requirement::PythonRequirement; +use crate::version_map::VersionMap; +use crate::yanks::AllowedYanks; + +type VersionMapResponse = Result<(IndexUrl, BaseUrl, VersionMap), puffin_client::Error>; +type WheelMetadataResponse = Result<(Metadata21, Option), DistributionDatabaseError>; + +pub trait ResolverProvider: Send + Sync { + /// Get the version map for a package. + fn get_version_map<'io>( + &'io self, + package_name: &'io PackageName, + ) -> impl Future + Send + 'io; + + /// Get the metadata for a distribution. + /// + /// For a wheel, this is done by querying it's (remote) metadata, for a source dist we + /// (fetch and) build the source distribution and return the metadata from the built + /// distribution. + fn get_or_build_wheel_metadata<'io>( + &'io self, + dist: &'io Dist, + ) -> impl Future + Send + 'io; + + /// Set the [`Reporter`] to use for this installer. + #[must_use] + fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self; +} + +/// The main IO backend for the resolver, which does cached requests network requests using the +/// [`RegistryClient`] and [`DistributionDatabase`]. +pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> { + client: &'a RegistryClient, + fetcher: DistributionDatabase<'a, Context>, + tags: &'a Tags, + python_requirement: PythonRequirement<'a>, + exclude_newer: Option>, + allowed_yanks: AllowedYanks, +} + +impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Context> { + pub fn new( + client: &'a RegistryClient, + fetcher: DistributionDatabase<'a, Context>, + tags: &'a Tags, + python_requirement: PythonRequirement<'a>, + exclude_newer: Option>, + allowed_yanks: AllowedYanks, + ) -> Self { + Self { + client, + fetcher, + tags, + python_requirement, + exclude_newer, + allowed_yanks, + } + } +} + +impl<'a, Context: BuildContext + Send + Sync> ResolverProvider + for DefaultResolverProvider<'a, Context> +{ + fn get_version_map<'io>( + &'io self, + package_name: &'io PackageName, + ) -> impl Future + Send + 'io { + self.client + .simple(package_name) + .map_ok(move |(index, base, metadata)| { + ( + index, + base, + VersionMap::from_metadata( + metadata, + package_name, + self.tags, + &self.python_requirement, + &self.allowed_yanks, + self.exclude_newer.as_ref(), + ), + ) + }) + } + + fn get_or_build_wheel_metadata<'io>( + &'io self, + dist: &'io Dist, + ) -> impl Future + Send + 'io { + self.fetcher.get_or_build_wheel_metadata(dist) + } + + /// Set the [`puffin_distribution::Reporter`] to use for this installer. + #[must_use] + fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self { + Self { + fetcher: self.fetcher.with_reporter(reporter), + ..self + } + } +} diff --git a/crates/puffin-resolver/src/resolver/reporter.rs b/crates/puffin-resolver/src/resolver/reporter.rs new file mode 100644 index 000000000..12b257196 --- /dev/null +++ b/crates/puffin-resolver/src/resolver/reporter.rs @@ -0,0 +1,51 @@ +use std::sync::Arc; + +use url::Url; + +use distribution_types::{SourceDist, VersionOrUrl}; +use puffin_normalize::PackageName; + +pub type BuildId = usize; + +pub trait Reporter: Send + Sync { + /// Callback to invoke when a dependency is resolved. + fn on_progress(&self, name: &PackageName, version: VersionOrUrl); + + /// Callback to invoke when the resolution is complete. + fn on_complete(&self); + + /// Callback to invoke when a source distribution build is kicked off. + fn on_build_start(&self, dist: &SourceDist) -> usize; + + /// Callback to invoke when a source distribution build is complete. + fn on_build_complete(&self, dist: &SourceDist, id: usize); + + /// Callback to invoke when a repository checkout begins. + fn on_checkout_start(&self, url: &Url, rev: &str) -> usize; + + /// Callback to invoke when a repository checkout completes. + fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize); +} + +/// A facade for converting from [`Reporter`] to [`puffin_distribution::Reporter`]. +pub(crate) struct Facade { + pub(crate) reporter: Arc, +} + +impl puffin_distribution::Reporter for Facade { + fn on_build_start(&self, dist: &SourceDist) -> usize { + self.reporter.on_build_start(dist) + } + + fn on_build_complete(&self, dist: &SourceDist, id: usize) { + self.reporter.on_build_complete(dist, id); + } + + fn on_checkout_start(&self, url: &Url, rev: &str) -> usize { + self.reporter.on_checkout_start(url, rev) + } + + fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize) { + self.reporter.on_checkout_complete(url, rev, index); + } +}