Retry streaming Python and bin download errors (#15567)

When there is an error during the streaming download and unpack for
Python interpreter and bin installs, we would previously fail, causing a
lot of CI flakes on GitHub Actions.

The problem was that the error is not one of the extended IO errors we
were previously handling, but a regular reqwest error, nested below
layers of errors of other crates processing the stream, including some
IO errors. We now handle nested reqwest errors, too.

This surfaced another problem: Our manual retry loop couldn't inform the
retry middleware that it already performed the limit of retries, and
that the middleware should not retry anymore. While too many retries are
more a problem for debugging than for the user, this causes confusing
error output. To work around this, we disable the retries in the client
and handle all retry errors in our loop.

Fixes https://github.com/astral-sh/uv/issues/14171

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>

Cargo.lock

@@ -3358,21 +3358,21 @@ dependencies = [
 [[package]]
 name = "reqwest-middleware"
 version = "0.4.2"
-source = "git+https://github.com/astral-sh/reqwest-middleware?rev=ad8b9d332d1773fde8b4cd008486de5973e0a3f8#ad8b9d332d1773fde8b4cd008486de5973e0a3f8"
+source = "git+https://github.com/astral-sh/reqwest-middleware?rev=7650ed76215a962a96d94a79be71c27bffde7ab2#7650ed76215a962a96d94a79be71c27bffde7ab2"
 dependencies = [
  "anyhow",
  "async-trait",
  "http",
  "reqwest",
  "serde",
- "thiserror 1.0.69",
+ "thiserror 2.0.16",
  "tower-service",
 ]
 
 [[package]]
 name = "reqwest-retry"
 version = "0.7.0"
-source = "git+https://github.com/astral-sh/reqwest-middleware?rev=ad8b9d332d1773fde8b4cd008486de5973e0a3f8#ad8b9d332d1773fde8b4cd008486de5973e0a3f8"
+source = "git+https://github.com/astral-sh/reqwest-middleware?rev=7650ed76215a962a96d94a79be71c27bffde7ab2#7650ed76215a962a96d94a79be71c27bffde7ab2"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3383,7 +3383,7 @@ dependencies = [
  "reqwest",
  "reqwest-middleware",
  "retry-policies",
- "thiserror 1.0.69",
+ "thiserror 2.0.16",
  "tokio",
  "tracing",
  "wasmtimer",
@@ -4990,6 +4990,7 @@ dependencies = [
  "predicates",
  "regex",
  "reqwest",
+ "reqwest-retry",
  "rkyv",
  "rustc-hash",
  "self-replace",

Cargo.toml

@@ -149,8 +149,8 @@ reflink-copy = { version = "0.1.19" }
 regex = { version = "1.10.6" }
 regex-automata = { version = "0.4.8", default-features = false, features = ["dfa-build", "dfa-search", "perf", "std", "syntax"] }
 reqwest = { version = "0.12.22", default-features = false, features = ["json", "gzip", "deflate", "zstd", "stream", "system-proxy", "rustls-tls", "rustls-tls-native-roots", "socks", "multipart", "http2", "blocking"] }
-reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "ad8b9d332d1773fde8b4cd008486de5973e0a3f8", features = ["multipart"] }
+reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "7650ed76215a962a96d94a79be71c27bffde7ab2", features = ["multipart"] }
-reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "ad8b9d332d1773fde8b4cd008486de5973e0a3f8" }
+reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "7650ed76215a962a96d94a79be71c27bffde7ab2" }
 rkyv = { version = "0.8.8", features = ["bytecheck"] }
 rmp-serde = { version = "1.3.0" }
 rust-netrc = { version = "0.1.2" }
@@ -322,5 +322,5 @@ codegen-units = 1
 inherits = "release"
 
 [patch.crates-io]
-reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "ad8b9d332d1773fde8b4cd008486de5973e0a3f8" }
+reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "7650ed76215a962a96d94a79be71c27bffde7ab2" }
-reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "ad8b9d332d1773fde8b4cd008486de5973e0a3f8" }
+reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "7650ed76215a962a96d94a79be71c27bffde7ab2" }

crates/uv-bin-install/src/lib.rs

@@ -10,6 +10,7 @@ use std::time::{Duration, SystemTime};
 
 use futures::TryStreamExt;
 use reqwest_retry::RetryPolicy;
+use reqwest_retry::policies::ExponentialBackoff;
 use std::fmt;
 use thiserror::Error;
 use tokio::io::{AsyncRead, ReadBuf};
@@ -19,7 +20,7 @@ use url::Url;
 use uv_distribution_filename::SourceDistExtension;
 
 use uv_cache::{Cache, CacheBucket, CacheEntry};
-use uv_client::{BaseClient, is_extended_transient_error};
+use uv_client::{BaseClient, is_transient_network_error};
 use uv_extract::{Error as ExtractError, stream};
 use uv_pep440::Version;
 use uv_platform::Platform;
@@ -160,6 +161,7 @@ pub async fn bin_install(
     binary: Binary,
     version: &Version,
     client: &BaseClient,
+    retry_policy: &ExponentialBackoff,
     cache: &Cache,
     reporter: &dyn Reporter,
 ) -> Result<PathBuf, Error> {
@@ -195,6 +197,7 @@ pub async fn bin_install(
         binary,
         version,
         client,
+        retry_policy,
         cache,
         reporter,
         &platform_name,
@@ -227,6 +230,7 @@ async fn download_and_unpack_with_retry(
     binary: Binary,
     version: &Version,
     client: &BaseClient,
+    retry_policy: &ExponentialBackoff,
     cache: &Cache,
     reporter: &dyn Reporter,
     platform_name: &str,
@@ -237,7 +241,6 @@ async fn download_and_unpack_with_retry(
     let mut total_attempts = 0;
     let mut retried_here = false;
     let start_time = SystemTime::now();
-    let retry_policy = client.retry_policy();
 
     loop {
         let result = download_and_unpack(
@@ -259,7 +262,7 @@ async fn download_and_unpack_with_retry(
                 total_attempts += err.attempts();
                 let past_retries = total_attempts - 1;
 
-                if is_extended_transient_error(&err) {
+                if is_transient_network_error(&err) {
                     let retry_decision = retry_policy.should_retry(start_time, past_retries);
                     if let reqwest_retry::RetryDecision::Retry { execute_after } = retry_decision {
                         debug!(

crates/uv-client/src/base_client.rs

@@ -21,6 +21,7 @@ use reqwest_middleware::{ClientWithMiddleware, Middleware};
 use reqwest_retry::policies::ExponentialBackoff;
 use reqwest_retry::{
     DefaultRetryableStrategy, RetryTransientMiddleware, Retryable, RetryableStrategy,
+    default_on_request_error,
 };
 use thiserror::Error;
 use tracing::{debug, trace};
@@ -38,10 +39,10 @@ use uv_static::EnvVars;
 use uv_version::version;
 use uv_warnings::warn_user_once;
 
-use crate::Connectivity;
 use crate::linehaul::LineHaul;
 use crate::middleware::OfflineMiddleware;
 use crate::tls::read_identity;
+use crate::{Connectivity, WrappedReqwestError};
 
 /// Do not use this value directly outside tests, use [`retries_from_env`] instead.
 pub const DEFAULT_RETRIES: u32 = 3;
@@ -916,7 +917,7 @@ impl RetryableStrategy for UvRetryableStrategy {
             None | Some(Retryable::Fatal)
                 if res
                     .as_ref()
-                    .is_err_and(|err| is_extended_transient_error(err)) =>
+                    .is_err_and(|err| is_transient_network_error(err)) =>
             {
                 Some(Retryable::Transient)
             }
@@ -944,12 +945,15 @@ impl RetryableStrategy for UvRetryableStrategy {
     }
 }
 
-/// Check for additional transient error kinds not supported by the default retry strategy in `reqwest_retry`.
+/// Whether the error looks like a network error that should be retried.
 ///
-/// These cases should be safe to retry with [`Retryable::Transient`].
+/// There are two cases that the default retry strategy is missing:
-pub fn is_extended_transient_error(err: &dyn Error) -> bool {
+/// * Inside the reqwest middleware error is an `io::Error` such as a broken pipe
+/// * When streaming a response, a reqwest error may be hidden several layers behind errors
+///   of different crates processing the stream, including `io::Error` layers.
+pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
     // First, try to show a nice trace log
-    if let Some((Some(status), Some(url))) = find_source::<crate::WrappedReqwestError>(&err)
+    if let Some((Some(status), Some(url))) = find_source::<WrappedReqwestError>(&err)
         .map(|request_err| (request_err.status(), request_err.url()))
     {
         trace!("Considering retry of response HTTP {status} for {url}");
@@ -957,10 +961,39 @@ pub fn is_extended_transient_error(err: &dyn Error) -> bool {
         trace!("Considering retry of error: {err:?}");
     }
 
-    // IO Errors may be nested through custom IO errors.
+    let mut has_known_error = false;
-    let mut has_io_error = false;
+    // IO Errors or reqwest errors may be nested through custom IO errors or stream processing
-    for io_err in find_sources::<io::Error>(&err) {
+    // crates
-        has_io_error = true;
+    let mut current_source = err.source();
+    while let Some(source) = current_source {
+        if let Some(reqwest_err) = source.downcast_ref::<WrappedReqwestError>() {
+            has_known_error = true;
+            if let reqwest_middleware::Error::Reqwest(reqwest_err) = &**reqwest_err {
+                if default_on_request_error(reqwest_err) == Some(Retryable::Transient) {
+                    trace!("Retrying nested reqwest middleware error");
+                    return true;
+                }
+                if is_retryable_status_error(reqwest_err) {
+                    trace!("Retrying nested reqwest middleware status code error");
+                    return true;
+                }
+            }
+
+            trace!("Cannot retry nested reqwest middleware error");
+        } else if let Some(reqwest_err) = source.downcast_ref::<reqwest::Error>() {
+            has_known_error = true;
+            if default_on_request_error(reqwest_err) == Some(Retryable::Transient) {
+                trace!("Retrying nested reqwest error");
+                return true;
+            }
+            if is_retryable_status_error(reqwest_err) {
+                trace!("Retrying nested reqwest status code error");
+                return true;
+            }
+
+            trace!("Cannot retry nested reqwest error");
+        } else if let Some(io_err) = source.downcast_ref::<io::Error>() {
+            has_known_error = true;
             let retryable_io_err_kinds = [
                 // https://github.com/astral-sh/uv/issues/12054
                 io::ErrorKind::BrokenPipe,
@@ -977,18 +1010,34 @@ pub fn is_extended_transient_error(err: &dyn Error) -> bool {
                 trace!("Retrying error: `{}`", io_err.kind());
                 return true;
             }
+
             trace!(
                 "Cannot retry IO error `{}`, not a retryable IO error kind",
                 io_err.kind()
             );
         }
 
-    if !has_io_error {
+        current_source = source.source();
-        trace!("Cannot retry error: not an extended IO error");
+    }
+
+    if !has_known_error {
+        trace!("Cannot retry error: Neither an IO error nor a reqwest error");
     }
     false
 }
 
+/// Whether the error is a status code error that is retryable.
+///
+/// Port of `reqwest_retry::default_on_request_success`.
+fn is_retryable_status_error(reqwest_err: &reqwest::Error) -> bool {
+    let Some(status) = reqwest_err.status() else {
+        return false;
+    };
+    status.is_server_error()
+        || status == StatusCode::REQUEST_TIMEOUT
+        || status == StatusCode::TOO_MANY_REQUESTS
+}
+
 /// Find the first source error of a specific type.
 ///
 /// See <https://github.com/seanmonstar/reqwest/issues/1602#issuecomment-1220996681>
@@ -1003,15 +1052,6 @@ fn find_source<E: Error + 'static>(orig: &dyn Error) -> Option<&E> {
     None
 }
 
-/// Return all errors in the chain of a specific type.
-///
-/// This handles cases such as nested `io::Error`s.
-///
-/// See <https://github.com/seanmonstar/reqwest/issues/1602#issuecomment-1220996681>
-fn find_sources<E: Error + 'static>(orig: &dyn Error) -> impl Iterator<Item = &E> {
-    iter::successors(find_source::<E>(orig), |&err| find_source(err))
-}
-
 // TODO(konsti): Remove once we find a native home for `retries_from_env`
 #[derive(Debug, Error)]
 pub enum RetryParsingError {

crates/uv-client/src/cached_client.rs

@@ -14,7 +14,7 @@ use uv_fs::write_atomic;
 use uv_redacted::DisplaySafeUrl;
 
 use crate::BaseClient;
-use crate::base_client::is_extended_transient_error;
+use crate::base_client::is_transient_network_error;
 use crate::{
     Error, ErrorKind,
     httpcache::{AfterResponse, BeforeRequest, CachePolicy, CachePolicyBuilder},
@@ -141,7 +141,7 @@ impl<CallbackError: std::error::Error + 'static> CachedClientError<CallbackError
         }
     }
 
-    fn error(&self) -> &dyn std::error::Error {
+    fn error(&self) -> &(dyn std::error::Error + 'static) {
         match self {
             Self::Client { err, .. } => err,
             Self::Callback { err, .. } => err,
@@ -452,7 +452,8 @@ impl CachedClient {
         .await
     }
 
-    #[instrument(name="read_and_parse_cache", skip_all, fields(file = %cache_entry.path().display()))]
+    #[instrument(name = "read_and_parse_cache", skip_all, fields(file = %cache_entry.path().display()
+    ))]
     async fn read_cache(cache_entry: &CacheEntry) -> Option<DataWithCachePolicy> {
         match DataWithCachePolicy::from_path_async(cache_entry.path()).await {
             Ok(data) => Some(data),
@@ -680,7 +681,7 @@ impl CachedClient {
 
             if result
                 .as_ref()
-                .is_err_and(|err| is_extended_transient_error(err.error()))
+                .is_err_and(|err| is_transient_network_error(err.error()))
             {
                 // If middleware already retried, consider that in our retry budget
                 let total_retries = past_retries + middleware_retries;
@@ -739,7 +740,7 @@ impl CachedClient {
             if result
                 .as_ref()
                 .err()
-                .is_some_and(|err| is_extended_transient_error(err.error()))
+                .is_some_and(|err| is_transient_network_error(err.error()))
             {
                 let total_retries = past_retries + middleware_retries;
                 let retry_decision = retry_policy.should_retry(start_time, total_retries);

crates/uv-client/src/lib.rs

@@ -1,7 +1,7 @@
 pub use base_client::{
     AuthIntegration, BaseClient, BaseClientBuilder, DEFAULT_RETRIES, ExtraMiddleware,
     RedirectClientWithMiddleware, RequestBuilder, RetryParsingError, UvRetryableStrategy,
-    is_extended_transient_error, retries_from_env,
+    is_transient_network_error, retries_from_env,
 };
 pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy};
 pub use error::{Error, ErrorKind, WrappedReqwestError};

crates/uv-python/src/downloads.rs

@@ -12,6 +12,7 @@ use futures::TryStreamExt;
 use itertools::Itertools;
 use once_cell::sync::OnceCell;
 use owo_colors::OwoColorize;
+use reqwest_retry::policies::ExponentialBackoff;
 use reqwest_retry::{RetryError, RetryPolicy};
 use serde::Deserialize;
 use thiserror::Error;
@@ -21,7 +22,7 @@ use tokio_util::either::Either;
 use tracing::{debug, instrument};
 use url::Url;
 
-use uv_client::{BaseClient, WrappedReqwestError, is_extended_transient_error};
+use uv_client::{BaseClient, WrappedReqwestError, is_transient_network_error};
 use uv_distribution_filename::{ExtensionError, SourceDistExtension};
 use uv_extract::hash::Hasher;
 use uv_fs::{Simplified, rename_with_retry};
@@ -930,6 +931,7 @@ impl ManagedPythonDownload {
     pub async fn fetch_with_retry(
         &self,
         client: &BaseClient,
+        retry_policy: &ExponentialBackoff,
         installation_dir: &Path,
         scratch_dir: &Path,
         reinstall: bool,
@@ -940,7 +942,6 @@ impl ManagedPythonDownload {
         let mut total_attempts = 0;
         let mut retried_here = false;
         let start_time = SystemTime::now();
-        let retry_policy = client.retry_policy();
         loop {
             let result = self
                 .fetch(
@@ -961,7 +962,7 @@ impl ManagedPythonDownload {
                     total_attempts += err.attempts();
                     // We currently interpret e.g. "3 retries" to mean we should make 4 attempts.
                     let n_past_retries = total_attempts - 1;
-                    if is_extended_transient_error(&err) {
+                    if is_transient_network_error(&err) {
                         let retry_decision = retry_policy.should_retry(start_time, n_past_retries);
                         if let reqwest_retry::RetryDecision::Retry { execute_after } =
                             retry_decision

crates/uv-python/src/installation.rs

@@ -5,10 +5,11 @@ use std::str::FromStr;
 
 use indexmap::IndexMap;
 use ref_cast::RefCast;
+use reqwest_retry::policies::ExponentialBackoff;
 use tracing::{debug, info};
 
 use uv_cache::Cache;
-use uv_client::BaseClientBuilder;
+use uv_client::{BaseClientBuilder, retries_from_env};
 use uv_pep440::{Prerelease, Version};
 use uv_platform::{Arch, Libc, Os, Platform};
 use uv_preview::Preview;
@@ -228,12 +229,17 @@ impl PythonInstallation {
         let scratch_dir = installations.scratch();
         let _lock = installations.lock().await?;
 
-        let client = client_builder.build();
+        // Python downloads are performing their own retries to catch stream errors, disable the
+        // default retries to avoid the middleware from performing uncontrolled retries.
+        let retry_policy =
+            ExponentialBackoff::builder().build_with_max_retries(retries_from_env()?);
+        let client = client_builder.clone().retries(0).build();
 
         info!("Fetching requested Python...");
         let result = download
             .fetch_with_retry(
                 &client,
+                &retry_policy,
                 installations_dir,
                 &scratch_dir,
                 false,

crates/uv-python/src/lib.rs

@@ -99,6 +99,9 @@ pub enum Error {
 
     #[error(transparent)]
     InvalidEnvironment(#[from] environment::InvalidEnvironment),
+
+    #[error(transparent)]
+    RetryParsing(#[from] uv_client::RetryParsingError),
 }
 
 impl Error {

crates/uv/Cargo.toml

@@ -88,6 +88,7 @@ owo-colors = { workspace = true }
 petgraph = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
+reqwest-retry = { workspace = true }
 rkyv = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }

crates/uv/src/commands/project/format.rs

@@ -2,11 +2,12 @@ use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{Context, Result};
+use reqwest_retry::policies::ExponentialBackoff;
 use tokio::process::Command;
 
 use uv_bin_install::{Binary, bin_install};
 use uv_cache::Cache;
-use uv_client::BaseClientBuilder;
+use uv_client::{BaseClientBuilder, retries_from_env};
 use uv_pep440::Version;
 use uv_preview::{Preview, PreviewFeatures};
 use uv_warnings::warn_user;
@@ -45,13 +46,23 @@ pub(crate) async fn format(
     // Parse version if provided
     let version = version.as_deref().map(Version::from_str).transpose()?;
 
-    let client = client_builder.build();
+    // Python downloads are performing their own retries to catch stream errors, disable the
+    // default retries to avoid the middleware from performing uncontrolled retries.
+    let retry_policy = ExponentialBackoff::builder().build_with_max_retries(retries_from_env()?);
+    let client = client_builder.retries(0).build();
 
     // Get the path to Ruff, downloading it if necessary
     let reporter = BinaryDownloadReporter::single(printer);
     let default_version = Binary::Ruff.default_version();
     let version = version.as_ref().unwrap_or(&default_version);
-    let ruff_path = bin_install(Binary::Ruff, version, &client, &cache, &reporter)
+    let ruff_path = bin_install(
+        Binary::Ruff,
+        version,
+        &client,
+        &retry_policy,
+        &cache,
+        &reporter,
+    )
     .await
     .with_context(|| format!("Failed to install ruff {version}"))?;
 

crates/uv/src/commands/python/install.rs

@@ -11,9 +11,11 @@ use futures::stream::FuturesUnordered;
 use indexmap::IndexSet;
 use itertools::{Either, Itertools};
 use owo_colors::{AnsiColors, OwoColorize};
+use reqwest_retry::policies::ExponentialBackoff;
 use rustc_hash::{FxHashMap, FxHashSet};
 use tracing::{debug, trace};
-use uv_client::BaseClientBuilder;
+
+use uv_client::{BaseClientBuilder, retries_from_env};
 use uv_fs::Simplified;
 use uv_platform::{Arch, Libc};
 use uv_preview::{Preview, PreviewFeatures};
@@ -401,8 +403,11 @@ pub(crate) async fn install(
         .unique_by(|download| download.key())
         .collect::<Vec<_>>();
 
-    // Download and unpack the Python versions concurrently
+    // Python downloads are performing their own retries to catch stream errors, disable the
-    let client = client_builder.build();
+    // default retries to avoid the middleware from performing uncontrolled retries.
+    let retry_policy = ExponentialBackoff::builder().build_with_max_retries(retries_from_env()?);
+    let client = client_builder.retries(0).build();
+
     let reporter = PythonDownloadReporter::new(printer, downloads.len() as u64);
     let mut tasks = FuturesUnordered::new();
 
@@ -413,6 +418,7 @@ pub(crate) async fn install(
                 download
                     .fetch_with_retry(
                         &client,
+                        &retry_policy,
                         installations_dir,
                         &scratch_dir,
                         reinstall,

crates/uv/tests/it/network.rs

@@ -293,8 +293,8 @@ async fn python_install_io_error() {
 
     ----- stderr -----
     error: Failed to install cpython-3.10.0-macos-aarch64-none
-      Caused by: Failed to download [SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst
       Caused by: Request failed after 3 retries
+      Caused by: Failed to download [SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst
       Caused by: error sending request for url ([SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst)
       Caused by: client error (SendRequest)
       Caused by: connection closed before message completed