mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-12-04 00:54:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Share flat index across resolutions (#930)

Browse source 
## Summary

This PR restructures the flat index fetching in a few ways:

1. It now lives in its own `FlatIndexClient`, since it felt a bit
awkward (in my opinion) for it to live in `RegistryClient`.
2. We now fetch the `FlatIndex` outside of the resolver. This has a few
benefits: (1) the resolver construct is no longer `async` and no longer
returns `Result`, which feels better for a resolver; and (2) we can
share the `FlatIndex` across resolutions rather than re-fetching it for
every source distribution build.
This commit is contained in:
Charlie Marsh 2024-01-15 11:02:02 -05:00 • committed by GitHub
parent e6d7124147
commit 42888a9609
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 336 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

56
crates/distribution-types/src/index_url.rs
View file

@ -101,10 +101,10 @@ impl Display for FlatIndexLocation {
}
}
/// The index URLs to use for fetching packages.
/// The index locations to use for fetching packages.
///
/// "pip treats all package sources equally" (<https://github.com/pypa/pip/issues/8606#issuecomment-788754817>),
/// and so do we, i.e. you can't rely that on any particular order of querying indices.
/// and so do we, i.e., you can't rely that on any particular order of querying indices.
///
/// If the fields are none and empty, ignore the package index, instead rely on local archives and
/// caches.
@ -118,7 +118,7 @@ pub struct IndexLocations {
}
impl Default for IndexLocations {
/// Just pypi
/// By default, use the `PyPI` index.
fn default() -> Self {
Self {
index: Some(IndexUrl::Pypi),
@ -137,6 +137,7 @@ impl IndexLocations {
no_index: bool,
) -> Self {
if no_index {
// TODO(charlie): Warn if the user passes in arguments here alongside `--no-index`.
Self {
index: None,
extra_index: Vec::new(),
@ -150,10 +151,6 @@ impl IndexLocations {
}
}
}
pub fn no_index(&self) -> bool {
self.index.is_none() && self.extra_index.is_empty()
}
}
impl<'a> IndexLocations {
@ -164,4 +161,49 @@ impl<'a> IndexLocations {
pub fn flat_indexes(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
self.flat_index.iter()
}
pub fn index_urls(&'a self) -> IndexUrls {
IndexUrls {
index: self.index.clone(),
extra_index: self.extra_index.clone(),
}
}
}
/// The index URLs to use for fetching packages.
///
/// From a pip perspective, this type merges `--index-url` and `--extra-index-url`.
#[derive(Debug, Clone)]
pub struct IndexUrls {
index: Option<IndexUrl>,
extra_index: Vec<IndexUrl>,
}
impl Default for IndexUrls {
/// By default, use the `PyPI` index.
fn default() -> Self {
Self {
index: Some(IndexUrl::Pypi),
extra_index: Vec::new(),
}
}
}
impl<'a> IndexUrls {
pub fn indexes(&'a self) -> impl Iterator<Item = &'a IndexUrl> + 'a {
self.index.iter().chain(self.extra_index.iter())
}
pub fn no_index(&self) -> bool {
self.index.is_none() && self.extra_index.is_empty()
}
}
impl From<IndexLocations> for IndexUrls {
fn from(locations: IndexLocations) -> Self {
Self {
index: locations.index,
extra_index: locations.extra_index,
}
}
}

21
crates/puffin-cli/src/commands/pip_compile.rs
View file

@ -18,7 +18,7 @@ use pep508_rs::Requirement;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::RegistryClientBuilder;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_installer::Downloader;
use puffin_interpreter::{Interpreter, PythonVersion};
@ -144,15 +144,23 @@ pub(crate) async fn pip_compile(
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_locations(index_locations.clone())
.index_urls(index_locations.index_urls())
.build();
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, &tags)
};
let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
let build_dispatch = BuildDispatch::new(
&client,
&cache,
&interpreter,
&index_locations,
&flat_index,
interpreter.sys_executable().to_path_buf(),
setup_py,
no_build,
@ -219,6 +227,13 @@ pub(crate) async fn pip_compile(
editable_metadata,
);
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, &tags)
};
// Resolve the dependencies.
let resolver = Resolver::new(
manifest,
@ -227,9 +242,9 @@ pub(crate) async fn pip_compile(
&interpreter,
&tags,
&client,
&flat_index,
&build_dispatch,
)
.await?
.with_reporter(ResolverReporter::from(printer));
let resolution = match resolver.resolve().await {
Err(puffin_resolver::ResolveError::NoSolution(err)) => {

16
crates/puffin-cli/src/commands/pip_install.rs
View file

@ -18,7 +18,7 @@ use pep508_rs::{MarkerEnvironment, Requirement};
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::{RegistryClient, RegistryClientBuilder};
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_installer::{
BuiltEditable, Downloader, InstallPlan, Reinstall, ResolvedEditable, SitePackages,
@ -134,9 +134,16 @@ pub(crate) async fn pip_install(
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_locations(index_locations.clone())
.index_urls(index_locations.index_urls())
.build();
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, tags)
};
let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
let build_dispatch = BuildDispatch::new(
@ -144,6 +151,7 @@ pub(crate) async fn pip_install(
&cache,
&interpreter,
&index_locations,
&flat_index,
venv.python_executable(),
setup_py,
no_build,
@ -183,6 +191,7 @@ pub(crate) async fn pip_install(
tags,
markers,
&client,
&flat_index,
&build_dispatch,
options,
printer,
@ -333,6 +342,7 @@ async fn resolve(
tags: &Tags,
markers: &MarkerEnvironment,
client: &RegistryClient,
flat_index: &FlatIndex,
build_dispatch: &BuildDispatch<'_>,
options: ResolutionOptions,
mut printer: Printer,
@ -378,9 +388,9 @@ async fn resolve(
interpreter,
tags,
client,
flat_index,
build_dispatch,
)
.await?
.with_reporter(ResolverReporter::from(printer));
let resolution = resolver.resolve().await?;

21
crates/puffin-cli/src/commands/pip_sync.rs
View file

@ -10,7 +10,7 @@ use install_wheel_rs::linker::LinkMode;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::{FlatIndex, RegistryClient, RegistryClientBuilder};
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_installer::{Downloader, InstallPlan, Reinstall, ResolvedEditable, SitePackages};
use puffin_interpreter::Virtualenv;
@ -60,15 +60,23 @@ pub(crate) async fn pip_sync(
// Prep the registry client.
let client = RegistryClientBuilder::new(cache.clone())
.index_locations(index_locations.clone())
.index_urls(index_locations.index_urls())
.build();
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, tags)
};
// Prep the build context.
let build_dispatch = BuildDispatch::new(
&client,
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
venv.python_executable(),
setup_py,
no_build,
@ -130,7 +138,7 @@ pub(crate) async fn pip_sync(
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_locations(index_locations.clone())
.index_urls(index_locations.index_urls())
.build();
// Resolve any registry-based requirements.
@ -139,7 +147,12 @@ pub(crate) async fn pip_sync(
} else {
let start = std::time::Instant::now();
let flat_index = FlatIndex::from_files(client.flat_index().await?, tags);
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, tags)
};
let wheel_finder =
puffin_resolver::DistFinder::new(tags, &client, venv.interpreter(), &flat_index)

27
crates/puffin-cli/src/commands/venv.rs
View file

@ -12,7 +12,7 @@ use distribution_types::{DistributionMetadata, IndexLocations, Name};
use pep508_rs::Requirement;
use platform_host::Platform;
use puffin_cache::Cache;
use puffin_client::RegistryClientBuilder;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::Interpreter;
use puffin_traits::{BuildContext, SetupPyStrategy};
@ -63,6 +63,14 @@ enum VenvError {
#[error("Failed to install seed packages")]
#[diagnostic(code(puffin::venv::seed))]
SeedError(#[source] anyhow::Error),
#[error("Failed to extract interpreter tags")]
#[diagnostic(code(puffin::venv::tags))]
TagsError(#[source] platform_host::PlatformError),
#[error("Failed to resolve `--find-links` entry")]
#[diagnostic(code(puffin::venv::flat_index))]
FlatIndexError(#[source] puffin_client::Error),
}
/// Create a virtual environment.
@ -114,15 +122,30 @@ async fn venv_impl(
// Install seed packages.
if seed {
// Extract the interpreter.
let interpreter = venv.interpreter();
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone()).build();
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let tags = interpreter.tags().map_err(VenvError::TagsError)?;
let client = FlatIndexClient::new(&client, cache);
let entries = client
.fetch(index_locations.flat_indexes())
.await
.map_err(VenvError::FlatIndexError)?;
FlatIndex::from_entries(entries, tags)
};
// Prep the build context.
let build_dispatch = BuildDispatch::new(
&client,
cache,
venv.interpreter(),
interpreter,
index_locations,
&flat_index,
venv.python_executable(),
SetupPyStrategy::default(),
true,

165
crates/puffin-client/src/flat_index.rs
View file

@ -1,19 +1,172 @@
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::path::PathBuf;
use reqwest::Response;
use rustc_hash::FxHashMap;
use tracing::instrument;
use tracing::{debug, info_span, instrument, warn, Instrument};
use url::Url;
use distribution_filename::DistFilename;
use distribution_types::{
BuiltDist, Dist, File, IndexUrl, PrioritizedDistribution, RegistryBuiltDist,
RegistrySourceDist, SourceDist,
BuiltDist, Dist, File, FileLocation, FlatIndexLocation, IndexUrl, PrioritizedDistribution,
RegistryBuiltDist, RegistrySourceDist, SourceDist,
};
use pep440_rs::Version;
use pep508_rs::VerbatimUrl;
use platform_tags::Tags;
use puffin_cache::{Cache, CacheBucket};
use puffin_normalize::PackageName;
use pypi_types::Hashes;
pub type FlatIndexEntry = (DistFilename, File, IndexUrl);
use crate::html::SimpleHtml;
use crate::{Error, RegistryClient};
type FlatIndexEntry = (DistFilename, File, IndexUrl);
/// A client for reading distributions from `--find-links` entries (either local directories or
/// remote HTML indexes).
#[derive(Debug, Clone)]
pub struct FlatIndexClient<'a> {
client: &'a RegistryClient,
cache: &'a Cache,
}
impl<'a> FlatIndexClient<'a> {
/// Create a new [`FlatIndexClient`].
pub fn new(client: &'a RegistryClient, cache: &'a Cache) -> Self {
Self { client, cache }
}
/// Read the directories and flat remote indexes from `--find-links`.
#[allow(clippy::result_large_err)]
pub async fn fetch(
&self,
indexes: impl Iterator<Item = &FlatIndexLocation>,
) -> Result<Vec<FlatIndexEntry>, Error> {
let mut dists = Vec::new();
// TODO(konstin): Parallelize reads over flat indexes.
for flat_index in indexes {
let index_dists = match flat_index {
FlatIndexLocation::Path(path) => {
Self::read_from_directory(path).map_err(Error::FindLinks)?
}
FlatIndexLocation::Url(url) => self.read_from_url(url).await?,
};
if index_dists.is_empty() {
warn!("No packages found in `--find-links` entry: {}", flat_index);
} else {
debug!(
"Found {} package{} in `--find-links` entry: {}",
index_dists.len(),
if index_dists.len() == 1 { "" } else { "s" },
flat_index
);
}
dists.extend(index_dists);
}
Ok(dists)
}
/// Read a flat remote index from a `--find-links` URL.
async fn read_from_url(&self, url: &Url) -> Result<Vec<FlatIndexEntry>, Error> {
let cache_entry = self.cache.entry(
CacheBucket::FlatIndex,
"html",
format!("{}.msgpack", cache_key::digest(&url.to_string())),
);
let cached_client = self.client.cached_client();
let flat_index_request = cached_client
.uncached()
.get(url.clone())
.header("Accept-Encoding", "gzip")
.header("Accept", "text/html")
.build()?;
let parse_simple_response = |response: Response| {
async {
let text = response.text().await?;
let SimpleHtml { base, files } = SimpleHtml::parse(&text, url)
.map_err(|err| Error::from_html_err(err, url.clone()))?;
let files: Vec<File> = files
.into_iter()
.filter_map(|file| {
match File::try_from(file, &base) {
Ok(file) => Some(file),
Err(err) => {
// Ignore files with unparseable version specifiers.
warn!("Skipping file in {url}: {err}");
None
}
}
})
.collect();
Ok(files)
}
.instrument(info_span!("parse_flat_index_html", url = % url))
};
let files = cached_client
.get_cached_with_callback(flat_index_request, &cache_entry, parse_simple_response)
.await?;
Ok(files
.into_iter()
.filter_map(|file| {
Some((
DistFilename::try_from_normalized_filename(&file.filename)?,
file,
IndexUrl::Url(url.clone()),
))
})
.collect())
}
/// Read a flat remote index from a `--find-links` directory.
fn read_from_directory(path: &PathBuf) -> Result<Vec<FlatIndexEntry>, std::io::Error> {
// Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?;
let url = Url::from_directory_path(&path).expect("URL is already absolute");
let url = VerbatimUrl::unknown(url);
let mut dists = Vec::new();
for entry in fs_err::read_dir(&path)? {
let entry = entry?;
let metadata = entry.metadata()?;
if !metadata.is_file() {
continue;
}
let Ok(filename) = entry.file_name().into_string() else {
warn!(
"Skipping non-UTF-8 filename in `--find-links` directory: {}",
entry.file_name().to_string_lossy()
);
continue;
};
let file = File {
dist_info_metadata: None,
filename: filename.to_string(),
hashes: Hashes { sha256: None },
requires_python: None,
size: None,
upload_time: None,
url: FileLocation::Path(entry.path().to_path_buf(), url.clone()),
yanked: None,
};
let Some(filename) = DistFilename::try_from_normalized_filename(&filename) else {
debug!(
"Ignoring `--find-links` entry (expected a wheel or source distribution filename): {}",
entry.path().display()
);
continue;
};
dists.push((filename, file, IndexUrl::Pypi));
}
Ok(dists)
}
}
/// A set of [`PrioritizedDistribution`] from a `--find-links` entry, indexed by [`PackageName`]
/// and [`Version`].
@ -23,11 +176,11 @@ pub struct FlatIndex(FxHashMap<PackageName, FlatDistributions>);
impl FlatIndex {
/// Collect all files from a `--find-links` target into a [`FlatIndex`].
#[instrument(skip_all)]
pub fn from_files(dists: Vec<FlatIndexEntry>, tags: &Tags) -> Self {
pub fn from_entries(entries: Vec<FlatIndexEntry>, tags: &Tags) -> Self {
let mut flat_index = FxHashMap::default();
// Collect compatible distributions.
for (filename, file, index) in dists {
for (filename, file, index) in entries {
let distributions = flat_index.entry(filename.name().clone()).or_default();
Self::add_file(distributions, file, filename, tags, index);
}

2
crates/puffin-client/src/lib.rs
View file

@ -1,6 +1,6 @@
pub use cached_client::{CachedClient, CachedClientError, DataWithCachePolicy};
pub use error::Error;
pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexEntry};
pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient};
pub use registry_client::{
read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, VersionFiles,
};

163
crates/puffin-client/src/registry_client.rs
View file

@ -1,7 +1,6 @@
use std::collections::BTreeMap;
use std::fmt::Debug;
use std::io;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::str::FromStr;
use async_http_range_reader::{AsyncHttpRangeReader, AsyncHttpRangeReaderError};
@ -18,24 +17,21 @@ use tracing::{debug, info_span, instrument, trace, warn, Instrument};
use url::Url;
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
use distribution_types::{
BuiltDist, File, FileLocation, FlatIndexLocation, IndexLocations, IndexUrl, Name,
};
use distribution_types::{BuiltDist, File, FileLocation, IndexUrl, IndexUrls, Name};
use install_wheel_rs::find_dist_info;
use pep440_rs::Version;
use pep508_rs::VerbatimUrl;
use puffin_cache::{Cache, CacheBucket, WheelCache};
use puffin_normalize::PackageName;
use pypi_types::{BaseUrl, Hashes, Metadata21, SimpleJson};
use pypi_types::{BaseUrl, Metadata21, SimpleJson};
use crate::html::SimpleHtml;
use crate::remote_metadata::wheel_metadata_from_remote_zip;
use crate::{CachedClient, CachedClientError, Error, FlatIndexEntry};
use crate::{CachedClient, CachedClientError, Error};
/// A builder for an [`RegistryClient`].
#[derive(Debug, Clone)]
pub struct RegistryClientBuilder {
index_locations: IndexLocations,
index_urls: IndexUrls,
retries: u32,
cache: Cache,
}
@ -43,7 +39,7 @@ pub struct RegistryClientBuilder {
impl RegistryClientBuilder {
pub fn new(cache: Cache) -> Self {
Self {
index_locations: IndexLocations::default(),
index_urls: IndexUrls::default(),
cache,
retries: 3,
}
@ -52,8 +48,8 @@ impl RegistryClientBuilder {
impl RegistryClientBuilder {
#[must_use]
pub fn index_locations(mut self, index_urls: IndexLocations) -> Self {
self.index_locations = index_urls;
pub fn index_urls(mut self, index_urls: IndexUrls) -> Self {
self.index_urls = index_urls;
self
}
@ -76,7 +72,7 @@ impl RegistryClientBuilder {
.pool_max_idle_per_host(20)
.timeout(std::time::Duration::from_secs(60 * 5));
client_core.build().expect("Fail to build HTTP client.")
client_core.build().expect("Failed to build HTTP client.")
};
let retry_policy = ExponentialBackoff::builder().build_with_max_retries(self.retries);
@ -88,7 +84,7 @@ impl RegistryClientBuilder {
let client = CachedClient::new(uncached_client.clone());
RegistryClient {
index_locations: self.index_locations,
index_urls: self.index_urls,
client_raw: client_raw.clone(),
cache: self.cache,
client,
@ -100,7 +96,7 @@ impl RegistryClientBuilder {
#[derive(Debug, Clone)]
pub struct RegistryClient {
/// The index URLs to use for fetching packages.
index_locations: IndexLocations,
index_urls: IndexUrls,
/// The underlying HTTP client.
client: CachedClient,
/// Don't use this client, it only exists because `async_http_range_reader` needs
@ -116,133 +112,6 @@ impl RegistryClient {
&self.client
}
/// Read the directories and flat remote indexes from `--find-links`.
#[allow(clippy::result_large_err)]
pub async fn flat_index(&self) -> Result<Vec<FlatIndexEntry>, Error> {
let mut dists = Vec::new();
// TODO(konstin): Parallelize reads over flat indexes.
for flat_index in self.index_locations.flat_indexes() {
let index_dists = match flat_index {
FlatIndexLocation::Path(path) => {
Self::read_flat_index_dir(path).map_err(Error::FindLinks)?
}
FlatIndexLocation::Url(url) => self.read_flat_url(url).await?,
};
if index_dists.is_empty() {
warn!("No packages found in `--find-links` entry: {}", flat_index);
} else {
debug!(
"Found {} package{} in `--find-links` entry: {}",
index_dists.len(),
if index_dists.len() == 1 { "" } else { "s" },
flat_index
);
}
dists.extend(index_dists);
}
Ok(dists)
}
/// Read a flat remote index from a `--find-links` URL.
async fn read_flat_url(&self, url: &Url) -> Result<Vec<FlatIndexEntry>, Error> {
let cache_entry = self.cache.entry(
CacheBucket::FlatIndex,
"html",
format!("{}.msgpack", cache_key::digest(&url.to_string())),
);
let flat_index_request = self
.client
.uncached()
.get(url.clone())
.header("Accept-Encoding", "gzip")
.header("Accept", "text/html")
.build()?;
let parse_simple_response = |response: Response| {
async {
let text = response.text().await?;
let SimpleHtml { base, files } = SimpleHtml::parse(&text, url)
.map_err(|err| Error::from_html_err(err, url.clone()))?;
let files: Vec<File> = files
.into_iter()
.filter_map(|file| {
match File::try_from(file, &base) {
Ok(file) => Some(file),
Err(err) => {
// Ignore files with unparseable version specifiers.
warn!("Skipping file in {url}: {err}");
None
}
}
})
.collect();
Ok(files)
}
.instrument(info_span!("parse_flat_index_html", url = % url))
};
let files = self
.client
.get_cached_with_callback(flat_index_request, &cache_entry, parse_simple_response)
.await?;
Ok(files
.into_iter()
.filter_map(|file| {
Some((
DistFilename::try_from_normalized_filename(&file.filename)?,
file,
IndexUrl::Url(url.clone()),
))
})
.collect())
}
/// Read a flat remote index from a `--find-links` directory.
fn read_flat_index_dir(path: &PathBuf) -> Result<Vec<FlatIndexEntry>, io::Error> {
// Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?;
let url = Url::from_directory_path(&path).expect("URL is already absolute");
let url = VerbatimUrl::unknown(url);
let mut dists = Vec::new();
for entry in fs_err::read_dir(&path)? {
let entry = entry?;
let metadata = entry.metadata()?;
if !metadata.is_file() {
continue;
}
let Ok(filename) = entry.file_name().into_string() else {
warn!(
"Skipping non-UTF-8 filename in `--find-links` directory: {}",
entry.file_name().to_string_lossy()
);
continue;
};
let file = File {
dist_info_metadata: None,
filename: filename.to_string(),
hashes: Hashes { sha256: None },
requires_python: None,
size: None,
upload_time: None,
url: FileLocation::Path(entry.path().to_path_buf(), url.clone()),
yanked: None,
};
let Some(filename) = DistFilename::try_from_normalized_filename(&filename) else {
debug!(
"Ignoring `--find-links` entry (expected a wheel or source distribution filename): {}",
entry.path().display()
);
continue;
};
dists.push((filename, file, IndexUrl::Pypi));
}
Ok(dists)
}
/// Fetch a package from the `PyPI` simple API.
///
/// "simple" here refers to [PEP 503 – Simple Repository API](https://peps.python.org/pep-0503/)
@ -253,11 +122,11 @@ impl RegistryClient {
&self,
package_name: &PackageName,
) -> Result<(IndexUrl, SimpleMetadata), Error> {
if self.index_locations.no_index() {
if self.index_urls.no_index() {
return Err(Error::NoIndex(package_name.as_ref().to_string()));
}
for index in self.index_locations.indexes() {
for index in self.index_urls.indexes() {
let result = self.simple_single_index(package_name, index).await?;
return match result {
@ -396,7 +265,7 @@ impl RegistryClient {
file: &File,
url: &Url,
) -> Result<Metadata21, Error> {
if self.index_locations.no_index() {
if self.index_urls.no_index() {
return Err(Error::NoIndex(file.filename.clone()));
}
@ -445,7 +314,7 @@ impl RegistryClient {
url: &'data Url,
cache_shard: WheelCache<'data>,
) -> Result<Metadata21, Error> {
if self.index_locations.no_index() {
if self.index_urls.no_index() {
return Err(Error::NoIndex(url.to_string()));
}
@ -511,7 +380,7 @@ impl RegistryClient {
&self,
url: &Url,
) -> Result<Box<dyn futures::AsyncRead + Unpin + Send + Sync>, Error> {
if self.index_locations.no_index() {
if self.index_urls.no_index() {
return Err(Error::NoIndex(url.to_string()));
}

4
crates/puffin-dev/src/build.rs
View file

@ -9,7 +9,7 @@ use distribution_types::IndexLocations;
use platform_host::Platform;
use puffin_build::{SourceBuild, SourceBuildContext};
use puffin_cache::{Cache, CacheArgs};
use puffin_client::RegistryClientBuilder;
use puffin_client::{FlatIndex, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::Virtualenv;
use puffin_traits::{BuildContext, BuildKind, SetupPyStrategy};
@ -55,6 +55,7 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
let venv = Virtualenv::from_env(platform, &cache)?;
let client = RegistryClientBuilder::new(cache.clone()).build();
let index_urls = IndexLocations::default();
let flat_index = FlatIndex::default();
let setup_py = SetupPyStrategy::default();
let build_dispatch = BuildDispatch::new(
@ -62,6 +63,7 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
&cache,
venv.interpreter(),
&index_urls,
&flat_index,
venv.python_executable(),
setup_py,
false,

2
crates/puffin-dev/src/install_many.rs
View file

@ -60,6 +60,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> {
let venv = Virtualenv::from_env(platform, &cache)?;
let client = RegistryClientBuilder::new(cache.clone()).build();
let index_locations = IndexLocations::default();
let flat_index = FlatIndex::default();
let setup_py = SetupPyStrategy::default();
let tags = venv.interpreter().tags()?;
@ -68,6 +69,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> {
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
venv.python_executable(),
setup_py,
args.no_build,

14
crates/puffin-dev/src/resolve_cli.rs
View file

@ -13,7 +13,7 @@ use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution
use pep508_rs::Requirement;
use platform_host::Platform;
use puffin_cache::{Cache, CacheArgs};
use puffin_client::RegistryClientBuilder;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::Virtualenv;
use puffin_resolver::{Manifest, ResolutionOptions, Resolver};
@ -58,14 +58,20 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
let index_locations =
IndexLocations::from_args(args.index_url, args.extra_index_url, args.find_links, false);
let client = RegistryClientBuilder::new(cache.clone())
.index_locations(index_locations.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, venv.interpreter().tags()?)
};
let build_dispatch = BuildDispatch::new(
&client,
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
venv.python_executable(),
SetupPyStrategy::default(),
args.no_build,
@ -80,9 +86,9 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
venv.interpreter(),
tags,
&client,
&flat_index,
&build_dispatch,
)
.await?;
);
let resolution_graph = resolver.resolve().await.with_context(|| {
format!(
"No solution found when resolving: {}",

4
crates/puffin-dev/src/resolve_many.rs
View file

@ -16,7 +16,7 @@ use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{Requirement, VersionOrUrl};
use platform_host::Platform;
use puffin_cache::{Cache, CacheArgs};
use puffin_client::{RegistryClient, RegistryClientBuilder};
use puffin_client::{FlatIndex, RegistryClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::Virtualenv;
use puffin_normalize::PackageName;
@ -74,6 +74,7 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {
let venv = Virtualenv::from_env(platform, &cache)?;
let client = RegistryClientBuilder::new(cache.clone()).build();
let index_locations = IndexLocations::default();
let flat_index = FlatIndex::default();
let setup_py = SetupPyStrategy::default();
let build_dispatch = BuildDispatch::new(
@ -81,6 +82,7 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
venv.python_executable(),
setup_py,
args.no_build,

10
crates/puffin-dispatch/src/lib.rs
View file

@ -13,7 +13,7 @@ use distribution_types::{CachedDist, DistributionId, IndexLocations, Name, Resol
use pep508_rs::Requirement;
use puffin_build::{SourceBuild, SourceBuildContext};
use puffin_cache::Cache;
use puffin_client::RegistryClient;
use puffin_client::{FlatIndex, RegistryClient};
use puffin_installer::{Downloader, InstallPlan, Installer, Reinstall, SitePackages};
use puffin_interpreter::{Interpreter, Virtualenv};
use puffin_resolver::{Manifest, ResolutionOptions, Resolver};
@ -26,6 +26,7 @@ pub struct BuildDispatch<'a> {
cache: &'a Cache,
interpreter: &'a Interpreter,
index_locations: &'a IndexLocations,
flat_index: &'a FlatIndex,
base_python: PathBuf,
setup_py: SetupPyStrategy,
no_build: bool,
@ -35,11 +36,13 @@ pub struct BuildDispatch<'a> {
}
impl<'a> BuildDispatch<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
client: &'a RegistryClient,
cache: &'a Cache,
interpreter: &'a Interpreter,
index_locations: &'a IndexLocations,
flat_index: &'a FlatIndex,
base_python: PathBuf,
setup_py: SetupPyStrategy,
no_build: bool,
@ -49,6 +52,7 @@ impl<'a> BuildDispatch<'a> {
cache,
interpreter,
index_locations,
flat_index,
base_python,
setup_py,
no_build,
@ -98,9 +102,9 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.interpreter,
tags,
self.client,
self.flat_index,
self,
)
.await?;
);
let graph = resolver.resolve().await.with_context(|| {
format!(
"No solution found when resolving: {}",

12
crates/puffin-resolver/src/resolver/mod.rs
View file

@ -77,19 +77,21 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvid
/// Initialize a new resolver using the default backend doing real requests.
///
/// Reads the flat index entries.
pub async fn new(
#[allow(clippy::too_many_arguments)]
pub fn new(
manifest: Manifest,
options: ResolutionOptions,
markers: &'a MarkerEnvironment,
interpreter: &'a Interpreter,
tags: &'a Tags,
client: &'a RegistryClient,
flat_index: &'a FlatIndex,
build_context: &'a Context,
) -> Result<Self, puffin_client::Error> {
) -> Self {
let provider = DefaultResolverProvider::new(
client,
DistributionDatabase::new(build_context.cache(), tags, client, build_context),
FlatIndex::from_files(client.flat_index().await?, tags),
flat_index,
tags,
PythonRequirement::new(interpreter, markers),
options.exclude_newer,
@ -99,13 +101,13 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvid
.chain(manifest.constraints.iter())
.collect(),
);
Ok(Self::new_custom_io(
Self::new_custom_io(
manifest,
options,
markers,
PythonRequirement::new(interpreter, markers),
provider,
))
)
}
}

4
crates/puffin-resolver/src/resolver/provider.rs
View file

@ -50,7 +50,7 @@ pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> {
/// The [`DistributionDatabase`] used to build source distributions.
fetcher: DistributionDatabase<'a, Context>,
/// These are the entries from `--find-links` that act as overrides for index responses.
flat_index: FlatIndex,
flat_index: &'a FlatIndex,
tags: &'a Tags,
python_requirement: PythonRequirement<'a>,
exclude_newer: Option<DateTime<Utc>>,
@ -62,7 +62,7 @@ impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Contex
pub fn new(
client: &'a RegistryClient,
fetcher: DistributionDatabase<'a, Context>,
flat_index: FlatIndex,
flat_index: &'a FlatIndex,
tags: &'a Tags,
python_requirement: PythonRequirement<'a>,
exclude_newer: Option<DateTime<Utc>>,

7
crates/puffin-resolver/tests/resolver.rs
View file

@ -15,7 +15,7 @@ use pep508_rs::{MarkerEnvironment, Requirement, StringVersion};
use platform_host::{Arch, Os, Platform};
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::RegistryClientBuilder;
use puffin_client::{FlatIndex, RegistryClientBuilder};
use puffin_interpreter::{Interpreter, Virtualenv};
use puffin_resolver::{
DisplayResolutionGraph, Manifest, PreReleaseMode, ResolutionGraph, ResolutionMode,
@ -100,6 +100,7 @@ async fn resolve(
tags: &Tags,
) -> Result<ResolutionGraph> {
let client = RegistryClientBuilder::new(Cache::temp()?).build();
let flat_index = FlatIndex::default();
let interpreter = Interpreter::artificial(
Platform::current()?,
markers.clone(),
@ -118,9 +119,9 @@ async fn resolve(
&interpreter,
tags,
&client,
&flat_index,
&build_context,
)
.await?;
);
Ok(resolver.resolve().await?)
}