mirror of
https://github.com/astral-sh/uv.git
synced 2025-11-14 17:59:46 +00:00
Add graceful fallback for Artifactory indexes (#1574)
## Summary There are more details in https://github.com/astral-sh/uv/issues/1370, but it looks like Artifactory servers have incorrect behavior when it comes to HTTP range requests, in that they return `Accept-Ranges: bytes`, but then incorrectly return 200 requests when you actually ask for a given range. This PR ensures that we fallback gracefully in this case. It's built on https://github.com/prefix-dev/async_http_range_reader/pull/5. Assuming that gets merged upstream, we can then remove the Git dependency. Closes https://github.com/astral-sh/uv/issues/1370. ## Test Plan `cargo run pip install requests -i https://killjoyuvbug.jfrog.io/artifactory/api/pypi/pypi/simple --verbose`
This commit is contained in:
parent
12fea1d058
commit
facc60f3a8
4 changed files with 48 additions and 17 deletions
5
Cargo.lock
generated
5
Cargo.lock
generated
|
|
@ -203,9 +203,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async_http_range_reader"
|
name = "async_http_range_reader"
|
||||||
version = "0.5.0"
|
version = "0.6.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ea8c52f8b749ec4e8665041001a31208afdae9ef88916d2edf1610deb8b3616e"
|
checksum = "5143aaae4ec035a5d7cfda666eab896fe5428a2a8ab09ca651a2dce3a8f06912"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bisection",
|
"bisection",
|
||||||
"futures",
|
"futures",
|
||||||
|
|
@ -213,6 +213,7 @@ dependencies = [
|
||||||
"itertools 0.12.1",
|
"itertools 0.12.1",
|
||||||
"memmap2 0.9.4",
|
"memmap2 0.9.4",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"reqwest-middleware",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ anstream = { version = "0.6.5" }
|
||||||
anyhow = { version = "1.0.79" }
|
anyhow = { version = "1.0.79" }
|
||||||
async-compression = { version = "0.4.6" }
|
async-compression = { version = "0.4.6" }
|
||||||
async-trait = { version = "0.1.77" }
|
async-trait = { version = "0.1.77" }
|
||||||
async_http_range_reader = { version = "0.5.0" }
|
async_http_range_reader = { version = "0.6.1" }
|
||||||
async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "d76801da0943de985254fc6255c0e476b57c5836", features = ["deflate"] }
|
async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "d76801da0943de985254fc6255c0e476b57c5836", features = ["deflate"] }
|
||||||
base64 = { version = "0.21.7" }
|
base64 = { version = "0.21.7" }
|
||||||
cachedir = { version = "0.3.1" }
|
cachedir = { version = "0.3.1" }
|
||||||
|
|
|
||||||
|
|
@ -155,4 +155,35 @@ impl ErrorKind {
|
||||||
|
|
||||||
ErrorKind::RequestMiddlewareError(err)
|
ErrorKind::RequestMiddlewareError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if the error is due to the server not supporting HTTP range requests.
|
||||||
|
pub(crate) fn is_http_range_requests_unsupported(&self) -> bool {
|
||||||
|
match self {
|
||||||
|
// The server doesn't support range requests (as reported by the `HEAD` check).
|
||||||
|
ErrorKind::AsyncHttpRangeReader(
|
||||||
|
AsyncHttpRangeReaderError::HttpRangeRequestUnsupported,
|
||||||
|
) => {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The server doesn't support range requests, but we only discovered this while
|
||||||
|
// unzipping due to erroneous server behavior.
|
||||||
|
ErrorKind::Zip(_, ZipError::UpstreamReadError(err)) => {
|
||||||
|
if let Some(inner) = err.get_ref() {
|
||||||
|
if let Some(inner) = inner.downcast_ref::<AsyncHttpRangeReaderError>() {
|
||||||
|
if matches!(
|
||||||
|
inner,
|
||||||
|
AsyncHttpRangeReaderError::HttpRangeRequestUnsupported
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ use std::fmt::Debug;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
use async_http_range_reader::{AsyncHttpRangeReader, AsyncHttpRangeReaderError};
|
use async_http_range_reader::AsyncHttpRangeReader;
|
||||||
use async_zip::tokio::read::seek::ZipFileReader;
|
use async_zip::tokio::read::seek::ZipFileReader;
|
||||||
use futures::{FutureExt, TryStreamExt};
|
use futures::{FutureExt, TryStreamExt};
|
||||||
use reqwest::{Client, ClientBuilder, Response, StatusCode};
|
use reqwest::{Client, ClientBuilder, Response, StatusCode};
|
||||||
|
|
@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
|
||||||
use tempfile::tempfile_in;
|
use tempfile::tempfile_in;
|
||||||
use tokio::io::BufWriter;
|
use tokio::io::BufWriter;
|
||||||
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
||||||
use tracing::{debug, info_span, instrument, trace, warn, Instrument};
|
use tracing::{info_span, instrument, trace, warn, Instrument};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
|
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
|
||||||
|
|
@ -454,19 +454,17 @@ impl RegistryClient {
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(metadata) => return Ok(metadata),
|
Ok(metadata) => return Ok(metadata),
|
||||||
Err(err) => match err.into_kind() {
|
Err(err) => {
|
||||||
ErrorKind::AsyncHttpRangeReader(
|
if err.kind().is_http_range_requests_unsupported() {
|
||||||
AsyncHttpRangeReaderError::HttpRangeRequestUnsupported,
|
// The range request version failed. Fall back to downloading the entire file
|
||||||
) => {}
|
// and the reading the file from the zip the regular way.
|
||||||
kind => return Err(kind.into()),
|
warn!("Range requests not supported for {filename}; downloading wheel");
|
||||||
},
|
} else {
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The range request version failed (this is bad, the webserver should support this), fall
|
|
||||||
// back to downloading the entire file and the reading the file from the zip the regular
|
|
||||||
// way.
|
|
||||||
|
|
||||||
debug!("Range requests not supported for {filename}; downloading wheel");
|
|
||||||
// TODO(konstin): Download the wheel into a cache shared with the installer instead
|
// TODO(konstin): Download the wheel into a cache shared with the installer instead
|
||||||
// Note that this branch is only hit when you're not using and the server where
|
// Note that this branch is only hit when you're not using and the server where
|
||||||
// you host your wheels for some reasons doesn't support range requests
|
// you host your wheels for some reasons doesn't support range requests
|
||||||
|
|
@ -708,8 +706,9 @@ pub enum Connectivity {
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
use pypi_types::{JoinRelativeError, SimpleJson};
|
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
use pypi_types::{JoinRelativeError, SimpleJson};
|
||||||
use uv_normalize::PackageName;
|
use uv_normalize::PackageName;
|
||||||
|
|
||||||
use crate::{html::SimpleHtml, SimpleMetadata, SimpleMetadatum};
|
use crate::{html::SimpleHtml, SimpleMetadata, SimpleMetadatum};
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue