mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Add an --offline mode (#1270)

Browse source 
## Summary

This PR adds an `--offline` flag to Puffin that disables network
requests (implemented as a Reqwest middleware on our registry client).
When `--offline` is provided, we also allow the HTTP cache to return
stale data.

Closes #942.
This commit is contained in:
Charlie Marsh 2024-02-12 22:35:23 -05:00 • committed by GitHub
parent 942e353f65
commit 16bb80132f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 559 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/puffin-client/Cargo.toml
View file

@ -16,6 +16,7 @@ puffin-fs = { path = "../puffin-fs", features = ["tokio"] }
puffin-normalize = { path = "../puffin-normalize" }
pypi-types = { path = "../pypi-types" }
async-trait = { workspace = true }
async_http_range_reader = { workspace = true }
async_zip = { workspace = true, features = ["tokio"] }
chrono = { workspace = true }
@ -32,6 +33,7 @@ rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
task-local-extensions = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tl = { workspace = true }

24
crates/puffin-client/src/cached_client.rs
View file

@ -128,6 +128,8 @@ pub enum CacheControl {
None,
/// Apply `max-age=0, must-revalidate` to the request.
MustRevalidate,
/// Allow the client to return stale responses.
AllowStale,
}
impl From<Freshness> for CacheControl {
@ -307,7 +309,7 @@ impl CachedClient {
) -> Result<CachedResponse, Error> {
// Apply the cache control header, if necessary.
match cache_control {
CacheControl::None => {}
CacheControl::None | CacheControl::AllowStale => {}
CacheControl::MustRevalidate => {
req.headers_mut().insert(
http::header::CACHE_CONTROL,
@ -320,11 +322,17 @@ impl CachedClient {
debug!("Found fresh response for: {}", req.url());
CachedResponse::FreshCache(cached)
}
BeforeRequest::Stale(new_cache_policy_builder) => {
debug!("Found stale response for: {}", req.url());
self.send_cached_handle_stale(req, cached, new_cache_policy_builder)
.await?
}
BeforeRequest::Stale(new_cache_policy_builder) => match cache_control {
CacheControl::None | CacheControl::MustRevalidate => {
debug!("Found stale response for: {}", req.url());
self.send_cached_handle_stale(req, cached, new_cache_policy_builder)
.await?
}
CacheControl::AllowStale => {
debug!("Found stale (but allowed) response for: {}", req.url());
CachedResponse::FreshCache(cached)
}
},
BeforeRequest::NoMatch => {
// This shouldn't happen; if it does, we'll override the cache.
warn!(
@ -349,7 +357,7 @@ impl CachedClient {
.execute(req)
.instrument(info_span!("revalidation_request", url = url.as_str()))
.await
.map_err(ErrorKind::RequestMiddlewareError)?
.map_err(ErrorKind::from_middleware)?
.error_for_status()
.map_err(ErrorKind::RequestError)?;
match cached
@ -384,7 +392,7 @@ impl CachedClient {
.0
.execute(req)
.await
.map_err(ErrorKind::RequestMiddlewareError)?
.map_err(ErrorKind::from_middleware)?
.error_for_status()
.map_err(ErrorKind::RequestError)?;
let cache_policy = cache_policy_builder.build(&response);

20
crates/puffin-client/src/error.rs
View file

@ -6,6 +6,7 @@ use distribution_filename::{WheelFilename, WheelFilenameError};
use puffin_normalize::PackageName;
use crate::html;
use crate::middleware::OfflineError;
#[derive(Debug, thiserror::Error)]
#[error(transparent)]
@ -135,4 +136,23 @@ pub enum ErrorKind {
#[error("Writing to cache archive failed: {0}")]
ArchiveWrite(#[source] crate::rkyvutil::SerializerError),
#[error("Network connectivity is disabled, but the requested data wasn't found in the cache for: `{0}`")]
Offline(String),
}
impl ErrorKind {
pub(crate) fn from_middleware(err: reqwest_middleware::Error) -> Self {
if let reqwest_middleware::Error::Middleware(ref underlying) = err {
if let Some(err) = underlying.downcast_ref::<OfflineError>() {
return ErrorKind::Offline(err.url().to_string());
}
}
if let reqwest_middleware::Error::Reqwest(err) = err {
return ErrorKind::RequestError(err);
}
ErrorKind::RequestMiddlewareError(err)
}
}

27
crates/puffin-client/src/flat_index.rs
View file

@ -21,7 +21,7 @@ use pypi_types::Hashes;
use crate::cached_client::{CacheControl, CachedClientError};
use crate::html::SimpleHtml;
use crate::{Error, ErrorKind, RegistryClient};
use crate::{Connectivity, Error, ErrorKind, RegistryClient};
#[derive(Debug, thiserror::Error)]
pub enum FlatIndexError {
@ -92,11 +92,14 @@ impl<'a> FlatIndexClient<'a> {
"html",
format!("{}.msgpack", cache_key::digest(&url.to_string())),
);
let cache_control = CacheControl::from(
self.cache
.freshness(&cache_entry, None)
.map_err(ErrorKind::Io)?,
);
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&cache_entry, None)
.map_err(ErrorKind::Io)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let cached_client = self.client.cached_client();
@ -131,14 +134,22 @@ impl<'a> FlatIndexClient<'a> {
.boxed()
.instrument(info_span!("parse_flat_index_html", url = % url))
};
let files = cached_client
let response = cached_client
.get_serde(
flat_index_request,
&cache_entry,
cache_control,
parse_simple_response,
)
.await?;
.await;
let files = match response {
Ok(files) => files,
Err(CachedClientError::Client(err)) if matches!(err.kind(), ErrorKind::Offline(_)) => {
warn!("Remote `--find-links` entry was not available in the cache: {url}");
vec![]
}
Err(err) => return Err(err.into()),
};
Ok(files
.into_iter()
.filter_map(|file| {

3
crates/puffin-client/src/lib.rs
View file

@ -2,7 +2,7 @@ pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithC
pub use error::{Error, ErrorKind};
pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError};
pub use registry_client::{
read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum,
Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum,
VersionFiles,
};
pub use rkyvutil::OwnedArchive;
@ -12,6 +12,7 @@ mod error;
mod flat_index;
mod html;
mod httpcache;
mod middleware;
mod registry_client;
mod remote_metadata;
mod rkyvutil;

47
crates/puffin-client/src/middleware.rs Normal file
View file

@ -0,0 +1,47 @@
use std::fmt::Debug;
use reqwest::{Request, Response};
use reqwest_middleware::{Middleware, Next};
use task_local_extensions::Extensions;
use url::Url;
/// A custom error type for the offline middleware.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct OfflineError {
url: Url,
}
impl OfflineError {
/// Returns the URL that caused the error.
pub fn url(&self) -> &Url {
&self.url
}
}
impl std::fmt::Display for OfflineError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Network connectivity is disabled, but the requested data wasn't found in the cache for: `{}`", self.url)
}
}
impl std::error::Error for OfflineError {}
/// A middleware that always returns an error indicating that the client is offline.
pub(crate) struct OfflineMiddleware;
#[async_trait::async_trait]
impl Middleware for OfflineMiddleware {
async fn handle(
&self,
req: Request,
_extensions: &mut Extensions,
_next: Next<'_>,
) -> reqwest_middleware::Result<Response> {
Err(reqwest_middleware::Error::Middleware(
OfflineError {
url: req.url().clone(),
}
.into(),
))
}
}

107
crates/puffin-client/src/registry_client.rs
View file

@ -26,6 +26,7 @@ use pypi_types::{Metadata21, SimpleJson};
use crate::cached_client::CacheControl;
use crate::html::SimpleHtml;
use crate::middleware::OfflineMiddleware;
use crate::remote_metadata::wheel_metadata_from_remote_zip;
use crate::rkyvutil::OwnedArchive;
use crate::{CachedClient, CachedClientError, Error, ErrorKind};
@ -35,6 +36,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind};
pub struct RegistryClientBuilder {
index_urls: IndexUrls,
retries: u32,
connectivity: Connectivity,
cache: Cache,
}
@ -43,6 +45,7 @@ impl RegistryClientBuilder {
Self {
index_urls: IndexUrls::default(),
cache,
connectivity: Connectivity::Online,
retries: 3,
}
}
@ -55,6 +58,12 @@ impl RegistryClientBuilder {
self
}
#[must_use]
pub fn connectivity(mut self, connectivity: Connectivity) -> Self {
self.connectivity = connectivity;
self
}
#[must_use]
pub fn retries(mut self, retries: u32) -> Self {
self.retries = retries;
@ -69,6 +78,7 @@ impl RegistryClientBuilder {
pub fn build(self) -> RegistryClient {
let client_raw = {
// Disallow any connections.
let client_core = ClientBuilder::new()
.user_agent("puffin")
.pool_max_idle_per_host(20)
@ -77,19 +87,26 @@ impl RegistryClientBuilder {
client_core.build().expect("Failed to build HTTP client.")
};
let retry_policy = ExponentialBackoff::builder().build_with_max_retries(self.retries);
let retry_strategy = RetryTransientMiddleware::new_with_policy(retry_policy);
let uncached_client = match self.connectivity {
Connectivity::Online => {
let retry_policy =
ExponentialBackoff::builder().build_with_max_retries(self.retries);
let retry_strategy = RetryTransientMiddleware::new_with_policy(retry_policy);
reqwest_middleware::ClientBuilder::new(client_raw.clone())
.with(retry_strategy)
.build()
}
Connectivity::Offline => reqwest_middleware::ClientBuilder::new(client_raw.clone())
.with(OfflineMiddleware)
.build(),
};
let uncached_client = reqwest_middleware::ClientBuilder::new(client_raw.clone())
.with(retry_strategy)
.build();
let client = CachedClient::new(uncached_client.clone());
RegistryClient {
index_urls: self.index_urls,
client_raw: client_raw.clone(),
cache: self.cache,
client,
connectivity: self.connectivity,
client_raw: client_raw.clone(),
client: CachedClient::new(uncached_client.clone()),
}
}
}
@ -106,6 +123,8 @@ pub struct RegistryClient {
client_raw: Client,
/// Used for the remote wheel METADATA cache
cache: Cache,
/// The connectivity mode to use.
connectivity: Connectivity,
}
impl RegistryClient {
@ -114,12 +133,17 @@ impl RegistryClient {
&self.client
}
/// Return the [`Connectivity`] mode used by this client.
pub fn connectivity(&self) -> Connectivity {
self.connectivity
}
/// Fetch a package from the `PyPI` simple API.
///
/// "simple" here refers to [PEP 503 – Simple Repository API](https://peps.python.org/pep-0503/)
/// and [PEP 691 – JSON-based Simple API for Python Package Indexes](https://peps.python.org/pep-0691/),
/// which the pypi json api approximately implements.
#[instrument("simple_api", skip_all, fields(package = %package_name))]
#[instrument("simple_api", skip_all, fields(package = % package_name))]
pub async fn simple(
&self,
package_name: &PackageName,
@ -134,6 +158,7 @@ impl RegistryClient {
return match result {
Ok(metadata) => Ok((index.clone(), metadata)),
Err(CachedClientError::Client(err)) => match err.into_kind() {
ErrorKind::Offline(_) => continue,
ErrorKind::RequestError(err) => {
if err.status() == Some(StatusCode::NOT_FOUND) {
continue;
@ -146,7 +171,12 @@ impl RegistryClient {
};
}
Err(ErrorKind::PackageNotFound(package_name.to_string()).into())
match self.connectivity {
Connectivity::Online => {
Err(ErrorKind::PackageNotFound(package_name.to_string()).into())
}
Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()),
}
}
async fn simple_single_index(
@ -171,11 +201,14 @@ impl RegistryClient {
}),
format!("{package_name}.rkyv"),
);
let cache_control = CacheControl::from(
self.cache
.freshness(&cache_entry, Some(package_name))
.map_err(ErrorKind::Io)?,
);
let cache_control = match self.connectivity {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&cache_entry, Some(package_name))
.map_err(ErrorKind::Io)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let simple_request = self
.client
@ -243,7 +276,7 @@ impl RegistryClient {
/// 1. From a [PEP 658](https://peps.python.org/pep-0658/) data-dist-info-metadata url
/// 2. From a remote wheel by partial zip reading
/// 3. From a (temp) download of a remote wheel (this is a fallback, the webserver should support range requests)
#[instrument(skip_all, fields(%built_dist))]
#[instrument(skip_all, fields(% built_dist))]
pub async fn wheel_metadata(&self, built_dist: &BuiltDist) -> Result<Metadata21, Error> {
let metadata = match &built_dist {
BuiltDist::Registry(wheel) => match &wheel.file.url {
@ -314,11 +347,14 @@ impl RegistryClient {
WheelCache::Index(index).remote_wheel_dir(filename.name.as_ref()),
format!("{}.msgpack", filename.stem()),
);
let cache_control = CacheControl::from(
self.cache
.freshness(&cache_entry, Some(&filename.name))
.map_err(ErrorKind::Io)?,
);
let cache_control = match self.connectivity {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&cache_entry, Some(&filename.name))
.map_err(ErrorKind::Io)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let response_callback = |response: Response| async {
let bytes = response.bytes().await.map_err(ErrorKind::RequestError)?;
@ -364,11 +400,14 @@ impl RegistryClient {
cache_shard.remote_wheel_dir(filename.name.as_ref()),
format!("{}.msgpack", filename.stem()),
);
let cache_control = CacheControl::from(
self.cache
.freshness(&cache_entry, Some(&filename.name))
.map_err(ErrorKind::Io)?,
);
let cache_control = match self.connectivity {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&cache_entry, Some(&filename.name))
.map_err(ErrorKind::Io)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
// This response callback is special, we actually make a number of subsequent requests to
// fetch the file from the remote zip.
@ -421,7 +460,8 @@ impl RegistryClient {
}
// The range request version failed (this is bad, the webserver should support this), fall
// back to downloading the entire file and the reading the file from the zip the regular way
// back to downloading the entire file and the reading the file from the zip the regular
// way.
debug!("Range requests not supported for {filename}; downloading wheel");
// TODO(konstin): Download the wheel into a cache shared with the installer instead
@ -462,7 +502,7 @@ impl RegistryClient {
}
/// It doesn't really fit into `puffin_client`, but it avoids cyclical crate dependencies.
pub async fn read_metadata_async(
async fn read_metadata_async(
filename: &WheelFilename,
debug_source: String,
reader: impl tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin,
@ -642,6 +682,15 @@ impl MediaType {
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum Connectivity {
/// Allow access to the network.
Online,
/// Do not allow access to the network.
Offline,
}
#[cfg(test)]
mod tests {
use std::str::FromStr;

31
crates/puffin-distribution/src/distribution_database.rs
View file

@ -13,7 +13,7 @@ use distribution_types::{
};
use platform_tags::Tags;
use puffin_cache::{Cache, CacheBucket, Timestamp, WheelCache};
use puffin_client::{CacheControl, CachedClientError, RegistryClient};
use puffin_client::{CacheControl, CachedClientError, Connectivity, RegistryClient};
use puffin_fs::metadata_if_exists;
use puffin_git::GitSource;
use puffin_traits::{BuildContext, NoBinary, NoBuild};
@ -58,7 +58,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
locks: Arc::new(Locks::default()),
client,
build_context,
builder: SourceDistCachedBuilder::new(build_context, client.cached_client(), tags),
builder: SourceDistCachedBuilder::new(build_context, client, tags),
}
}
@ -169,11 +169,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
};
let req = self.client.cached_client().uncached().get(url).build()?;
let cache_control = CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
);
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()
@ -232,11 +236,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
.uncached()
.get(wheel.url.raw().clone())
.build()?;
let cache_control = CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
);
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()

60
crates/puffin-distribution/src/source/mod.rs
View file

@ -24,7 +24,9 @@ use platform_tags::Tags;
use puffin_cache::{
ArchiveTimestamp, CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Freshness, WheelCache,
};
use puffin_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy};
use puffin_client::{
CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
};
use puffin_fs::{write_atomic, LockedFile};
use puffin_git::{Fetch, GitSource};
use puffin_traits::{BuildContext, BuildKind, NoBuild, SourceBuildTrait};
@ -42,7 +44,7 @@ mod manifest;
/// Fetch and build a source distribution from a remote source, or from a local cache.
pub struct SourceDistCachedBuilder<'a, T: BuildContext> {
build_context: &'a T,
cached_client: &'a CachedClient,
client: &'a RegistryClient,
reporter: Option<Arc<dyn Reporter>>,
tags: &'a Tags,
}
@ -55,11 +57,11 @@ pub(crate) const METADATA: &str = "metadata.msgpack";
impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
/// Initialize a [`SourceDistCachedBuilder`] from a [`BuildContext`].
pub fn new(build_context: &'a T, cached_client: &'a CachedClient, tags: &'a Tags) -> Self {
pub fn new(build_context: &'a T, client: &'a RegistryClient, tags: &'a Tags) -> Self {
Self {
build_context,
reporter: None,
cached_client,
client,
tags,
}
}
@ -251,12 +253,15 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
subdirectory: Option<&'data Path>,
) -> Result<BuiltWheelMetadata, Error> {
let cache_entry = cache_shard.entry(MANIFEST);
let cache_control = CacheControl::from(
self.build_context
.cache()
.freshness(&cache_entry, Some(source_dist.name()))
.map_err(Error::CacheRead)?,
);
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.build_context
.cache()
.freshness(&cache_entry, Some(source_dist.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let download = |response| {
async {
@ -275,9 +280,15 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.boxed()
.instrument(info_span!("download", source_dist = %source_dist))
};
let req = self.cached_client.uncached().get(url.clone()).build()?;
let req = self
.client
.cached_client()
.uncached()
.get(url.clone())
.build()?;
let manifest = self
.cached_client
.client
.cached_client()
.get_serde(req, &cache_entry, cache_control, download)
.await
.map_err(|err| match err {
@ -345,12 +356,15 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
subdirectory: Option<&'data Path>,
) -> Result<Metadata21, Error> {
let cache_entry = cache_shard.entry(MANIFEST);
let cache_control = CacheControl::from(
self.build_context
.cache()
.freshness(&cache_entry, Some(source_dist.name()))
.map_err(Error::CacheRead)?,
);
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.build_context
.cache()
.freshness(&cache_entry, Some(source_dist.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let download = |response| {
async {
@ -369,9 +383,15 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.boxed()
.instrument(info_span!("download", source_dist = %source_dist))
};
let req = self.cached_client.uncached().get(url.clone()).build()?;
let req = self
.client
.cached_client()
.uncached()
.get(url.clone())
.build()?;
let manifest = self
.cached_client
.client
.cached_client()
.get_serde(req, &cache_entry, cache_control, download)
.await
.map_err(|err| match err {

24
crates/puffin-resolver/src/pubgrub/report.rs
View file

@ -406,12 +406,16 @@ impl PubGrubReportFormatter<'_> {
index_locations.flat_index().peekable().peek().is_none();
if let PubGrubPackage::Package(name, ..) = package {
if let Some(UnavailablePackage::NoIndex) =
unavailable_packages.get(name)
{
if no_find_links {
hints.insert(PubGrubHint::NoIndex);
match unavailable_packages.get(name) {
Some(UnavailablePackage::NoIndex) => {
if no_find_links {
hints.insert(PubGrubHint::NoIndex);
}
}
Some(UnavailablePackage::Offline) => {
hints.insert(PubGrubHint::Offline);
}
_ => {}
}
}
}
@ -460,6 +464,8 @@ pub(crate) enum PubGrubHint {
/// Requirements were unavailable due to lookups in the index being disabled and no extra
/// index was provided via `--find-links`
NoIndex,
/// A package was not found in the registry, but
Offline,
}
impl std::fmt::Display for PubGrubHint {
@ -493,6 +499,14 @@ impl std::fmt::Display for PubGrubHint {
":".bold(),
)
}
PubGrubHint::Offline => {
write!(
f,
"{}{} Packages were unavailable because the network was disabled",
"hint".bold().cyan(),
":".bold(),
)
}
}
}
}

17
crates/puffin-resolver/src/resolver/mod.rs
View file

@ -69,8 +69,10 @@ pub(crate) enum UnavailableVersion {
/// The package is unavailable and cannot be used
#[derive(Debug, Clone)]
pub(crate) enum UnavailablePackage {
/// Index loopups were disabled (i.e. `--no-index`) and the package was not found in a flat index (i.e. from `--find-links`)
/// Index lookups were disabled (i.e., `--no-index`) and the package was not found in a flat index (i.e. from `--find-links`)
NoIndex,
/// Network requests were disabled (i.e., `--offline`), and the package was not found in the cache.
Offline,
/// The package was not found in the registry
NotFound,
}
@ -346,6 +348,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
UnavailablePackage::NoIndex => {
"was not found in the provided package locations"
}
UnavailablePackage::Offline => "was not found in the cache",
UnavailablePackage::NotFound => {
"was not found in the package registry"
}
@ -574,6 +577,12 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
return Ok(None);
}
VersionsResponse::Offline => {
self.unavailable_packages
.insert(package_name.clone(), UnavailablePackage::Offline);
return Ok(None);
}
VersionsResponse::NotFound => {
self.unavailable_packages
.insert(package_name.clone(), UnavailablePackage::NotFound);
@ -911,6 +920,12 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
return Ok(None);
}
VersionsResponse::Offline => {
self.unavailable_packages
.insert(package_name.clone(), UnavailablePackage::Offline);
return Ok(None);
}
VersionsResponse::NotFound => {
self.unavailable_packages
.insert(package_name.clone(), UnavailablePackage::NotFound);

23
crates/puffin-resolver/src/resolver/provider.rs
View file

@ -30,6 +30,8 @@ pub enum VersionsResponse {
NotFound,
/// The package was not found in the local registry
NoIndex,
/// The package was not found in the cache and the network is not available.
Offline,
}
pub trait ResolverProvider: Send + Sync {
@ -116,19 +118,15 @@ impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Contex
impl<'a, Context: BuildContext + Send + Sync> ResolverProvider
for DefaultResolverProvider<'a, Context>
{
fn index_locations(&self) -> &IndexLocations {
self.fetcher.index_locations()
}
/// Make a simple api request for the package and convert the result to a [`VersionMap`].
/// Make a "Simple API" request for the package and convert the result to a [`VersionMap`].
async fn get_package_versions<'io>(
&'io self,
package_name: &'io PackageName,
) -> PackageVersionsResult {
let result = self.client.simple(package_name).await;
// If the simple api request was successful, perform on the slow conversion to `VersionMap` on the tokio
// threadpool
// If the "Simple API" request was successful, convert to `VersionMap` on the Tokio
// threadpool, since it can be slow.
match result {
Ok((index, metadata)) => {
let self_send = self.inner.clone();
@ -164,6 +162,13 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider
Ok(VersionsResponse::NoIndex)
}
}
puffin_client::ErrorKind::Offline(_) => {
if let Some(flat_index) = self.flat_index.get(package_name).cloned() {
Ok(VersionsResponse::Found(VersionMap::from(flat_index)))
} else {
Ok(VersionsResponse::Offline)
}
}
kind => Err(kind.into()),
},
}
@ -173,6 +178,10 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider
self.fetcher.get_or_build_wheel_metadata(dist).await
}
fn index_locations(&self) -> &IndexLocations {
self.fetcher.index_locations()
}
/// Set the [`puffin_distribution::Reporter`] to use for this installer.
#[must_use]
fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self {

4
crates/puffin/src/commands/pip_compile.rs
View file

@ -20,7 +20,7 @@ use pep508_rs::Requirement;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_client::{Connectivity, FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_fs::Normalized;
use puffin_installer::{Downloader, NoBinary};
@ -59,6 +59,7 @@ pub(crate) async fn pip_compile(
include_find_links: bool,
index_locations: IndexLocations,
setup_py: SetupPyStrategy,
connectivity: Connectivity,
no_build: &NoBuild,
python_version: Option<PythonVersion>,
exclude_newer: Option<DateTime<Utc>>,
@ -195,6 +196,7 @@ pub(crate) async fn pip_compile(
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.connectivity(connectivity)
.build();
// Resolve the flat indexes from `--find-links`.

6
crates/puffin/src/commands/pip_install.rs
View file

@ -17,7 +17,9 @@ use pep508_rs::{MarkerEnvironment, Requirement};
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder};
use puffin_client::{
Connectivity, FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder,
};
use puffin_dispatch::BuildDispatch;
use puffin_fs::Normalized;
use puffin_installer::{
@ -52,6 +54,7 @@ pub(crate) async fn pip_install(
reinstall: &Reinstall,
link_mode: LinkMode,
setup_py: SetupPyStrategy,
connectivity: Connectivity,
no_build: &NoBuild,
no_binary: &NoBinary,
strict: bool,
@ -138,6 +141,7 @@ pub(crate) async fn pip_install(
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.connectivity(connectivity)
.build();
// Resolve the flat indexes from `--find-links`.

18
crates/puffin/src/commands/pip_sync.rs
View file

@ -10,7 +10,9 @@ use install_wheel_rs::linker::LinkMode;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder};
use puffin_client::{
Connectivity, FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder,
};
use puffin_dispatch::BuildDispatch;
use puffin_fs::Normalized;
use puffin_installer::{
@ -35,6 +37,7 @@ pub(crate) async fn pip_sync(
link_mode: LinkMode,
index_locations: IndexLocations,
setup_py: SetupPyStrategy,
connectivity: Connectivity,
no_build: &NoBuild,
no_binary: &NoBinary,
strict: bool,
@ -84,6 +87,7 @@ pub(crate) async fn pip_sync(
// Prep the registry client.
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.connectivity(connectivity)
.build();
// Resolve the flat indexes from `--find-links`.
@ -169,24 +173,12 @@ pub(crate) async fn pip_sync(
return Ok(ExitStatus::Success);
}
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
// Resolve any registry-based requirements.
let remote = if remote.is_empty() {
Vec::new()
} else {
let start = std::time::Instant::now();
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, tags)
};
let wheel_finder = puffin_resolver::DistFinder::new(
tags,
&client,

12
crates/puffin/src/commands/venv.rs
View file

@ -13,7 +13,7 @@ use distribution_types::{DistributionMetadata, IndexLocations, Name};
use pep508_rs::Requirement;
use platform_host::Platform;
use puffin_cache::Cache;
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_client::{Connectivity, FlatIndex, FlatIndexClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_fs::Normalized;
use puffin_installer::NoBinary;
@ -25,11 +25,12 @@ use crate::commands::ExitStatus;
use crate::printer::Printer;
/// Create a virtual environment.
#[allow(clippy::unnecessary_wraps)]
#[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)]
pub(crate) async fn venv(
path: &Path,
python_request: Option<&str>,
index_locations: &IndexLocations,
connectivity: Connectivity,
seed: bool,
exclude_newer: Option<DateTime<Utc>>,
cache: &Cache,
@ -39,6 +40,7 @@ pub(crate) async fn venv(
path,
python_request,
index_locations,
connectivity,
seed,
exclude_newer,
cache,
@ -74,10 +76,12 @@ enum VenvError {
}
/// Create a virtual environment.
#[allow(clippy::too_many_arguments)]
async fn venv_impl(
path: &Path,
python_request: Option<&str>,
index_locations: &IndexLocations,
connectivity: Connectivity,
seed: bool,
exclude_newer: Option<DateTime<Utc>>,
cache: &Cache,
@ -118,7 +122,9 @@ async fn venv_impl(
let interpreter = venv.interpreter();
// Instantiate a client.
let client = RegistryClientBuilder::new(cache.clone()).build();
let client = RegistryClientBuilder::new(cache.clone())
.connectivity(connectivity)
.build();
// Resolve the flat indexes from `--find-links`.
let flat_index = {

53
crates/puffin/src/main.rs
View file

@ -13,6 +13,7 @@ use tracing::instrument;
use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl};
use puffin_cache::{Cache, CacheArgs, Refresh};
use puffin_client::Connectivity;
use puffin_installer::{NoBinary, Reinstall};
use puffin_interpreter::PythonVersion;
use puffin_normalize::{ExtraName, PackageName};
@ -50,6 +51,7 @@ mod requirements;
#[derive(Parser)]
#[command(author, version, about)]
#[command(propagate_version = true)]
#[allow(clippy::struct_excessive_bools)]
struct Cli {
#[command(subcommand)]
command: Commands,
@ -207,6 +209,15 @@ struct PipCompileArgs {
#[clap(long)]
no_header: bool,
/// Run offline, i.e., without accessing the network.
#[arg(
global = true,
long,
conflicts_with = "refresh",
conflicts_with = "refresh_package"
)]
offline: bool,
/// Refresh all cached data.
#[clap(long)]
refresh: bool,
@ -318,6 +329,15 @@ struct PipSyncArgs {
#[clap(long)]
reinstall_package: Vec<PackageName>,
/// Run offline, i.e., without accessing the network.
#[arg(
global = true,
long,
conflicts_with = "refresh",
conflicts_with = "refresh_package"
)]
offline: bool,
/// Refresh all cached data.
#[clap(long)]
refresh: bool,
@ -451,6 +471,15 @@ struct PipInstallArgs {
#[clap(long)]
reinstall_package: Vec<PackageName>,
/// Run offline, i.e., without accessing the network.
#[arg(
global = true,
long,
conflicts_with = "refresh",
conflicts_with = "refresh_package"
)]
offline: bool,
/// Refresh all cached data.
#[clap(long)]
refresh: bool,
@ -620,6 +649,10 @@ struct VenvArgs {
#[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")]
no_index: bool,
/// Run offline, i.e., without accessing the network.
#[arg(global = true, long)]
offline: bool,
/// Limit candidate packages to those that were uploaded prior to the given date.
///
/// Accepts both RFC 3339 timestamps (e.g., `2006-12-02T02:07:43Z`) and UTC dates in the same
@ -794,6 +827,11 @@ async fn run() -> Result<ExitStatus> {
} else {
SetupPyStrategy::Pep517
},
if args.offline {
Connectivity::Offline
} else {
Connectivity::Online
},
&no_build,
args.python_version,
args.exclude_newer,
@ -832,6 +870,11 @@ async fn run() -> Result<ExitStatus> {
} else {
SetupPyStrategy::Pep517
},
if args.offline {
Connectivity::Offline
} else {
Connectivity::Online
},
&no_build,
&no_binary,
args.strict,
@ -902,6 +945,11 @@ async fn run() -> Result<ExitStatus> {
} else {
SetupPyStrategy::Pep517
},
if args.offline {
Connectivity::Offline
} else {
Connectivity::Online
},
&no_build,
&no_binary,
args.strict,
@ -945,6 +993,11 @@ async fn run() -> Result<ExitStatus> {
&args.name,
args.python.as_deref(),
&index_locations,
if args.offline {
Connectivity::Offline
} else {
Connectivity::Online
},
args.seed,
args.exclude_newer,
&cache,

130
crates/puffin/tests/pip_compile.rs
View file

@ -3126,3 +3126,133 @@ fn conflicting_index_urls_requirements_txt() -> Result<()> {
Ok(())
}
/// Resolve without network access via the `--offline` flag.
#[test]
fn offline() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("black==23.10.1")?;
// Resolve with `--offline` with an empty cache.
puffin_snapshot!(context.compile()
.arg("requirements.in")
.arg("--offline"), @r###"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× No solution found when resolving dependencies:
╰─▶ Because black==23.10.1 was not found in the cache and you require
black==23.10.1, we can conclude that the requirements are unsatisfiable.
hint: Packages were unavailable because the network was disabled
"###
);
// Populate the cache.
puffin_snapshot!(context.compile()
.arg("requirements.in"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by Puffin v[VERSION] via the following command:
# puffin pip compile --cache-dir [CACHE_DIR] --exclude-newer 2023-11-18T12:00:00Z requirements.in
black==23.10.1
click==8.1.7
# via black
mypy-extensions==1.0.0
# via black
packaging==23.2
# via black
pathspec==0.11.2
# via black
platformdirs==4.0.0
# via black
----- stderr -----
Resolved 6 packages in [TIME]
"###
);
// Resolve with `--offline` with a populated cache.
puffin_snapshot!(context.compile()
.arg("requirements.in")
.arg("--offline"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by Puffin v[VERSION] via the following command:
# puffin pip compile --cache-dir [CACHE_DIR] --exclude-newer 2023-11-18T12:00:00Z requirements.in --offline
black==23.10.1
click==8.1.7
# via black
mypy-extensions==1.0.0
# via black
packaging==23.2
# via black
pathspec==0.11.2
# via black
platformdirs==4.0.0
# via black
----- stderr -----
Resolved 6 packages in [TIME]
"###
);
Ok(())
}
/// Resolve without network access via the `--offline` flag, using `--find-links` for an HTML
/// registry.
#[test]
fn offline_find_links() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("tqdm")?;
// Resolve with `--offline` and `--find-links`. We indicate that the network was disabled,
// since both the `--find-links` and the registry lookups fail (but, importantly, we don't error
// when failing to fetch the `--find-links` URL).
puffin_snapshot!(context.compile()
.arg("requirements.in")
.arg("--find-links")
.arg("https://download.pytorch.org/whl/torch_stable.html")
.arg("--offline"), @r###"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× No solution found when resolving dependencies:
╰─▶ Because tqdm was not found in the cache and you require tqdm, we can
conclude that the requirements are unsatisfiable.
hint: Packages were unavailable because the network was disabled
"###
);
// Resolve with `--offline`, `--find-links`, and `--no-index`.
// TODO(charlie): This should indicate that the network was disabled, but we don't "know" that
// the `--find-links` lookup failed.
puffin_snapshot!(context.compile()
.arg("requirements.in")
.arg("--find-links")
.arg("https://download.pytorch.org/whl/torch_stable.html")
.arg("--no-index")
.arg("--offline"), @r###"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× No solution found when resolving dependencies:
╰─▶ Because tqdm was not found in the provided package locations and you
require tqdm, we can conclude that the requirements are unsatisfiable.
"###
);
Ok(())
}

55
crates/puffin/tests/pip_sync.rs
View file

@ -2651,3 +2651,58 @@ fn find_links() -> Result<()> {
Ok(())
}
/// Install without network access via the `--offline` flag.
#[test]
fn offline() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("black==23.10.1")?;
// Install with `--offline` with an empty cache.
puffin_snapshot!(command(&context)
.arg("requirements.in")
.arg("--offline"), @r###"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: Network connectivity is disabled, but the requested data wasn't found in the cache for: `black`
"###
);
// Populate the cache.
puffin_snapshot!(command(&context)
.arg("requirements.in"), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
+ black==23.10.1
"###
);
// Install with `--offline` with a populated cache.
let venv = create_venv(&context.temp_dir, &context.cache_dir, "3.12");
puffin_snapshot!(command(&context)
.arg("requirements.in")
.arg("--offline")
.env("VIRTUAL_ENV", venv.as_os_str()), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Installed 1 package in [TIME]
+ black==23.10.1
"###
);
Ok(())
}