mirror of
https://github.com/astral-sh/uv.git
synced 2025-07-07 13:25:00 +00:00
Add support for alternate index URLs (#169)
As elsewhere, we just use the `pip` and `pip-compile` APIs. So we support `--index-url` to override PyPI, then `--extra-index-url` to add _additional_ indexes, and `--no-index` to avoid hitting the index at all. Closes #156.
This commit is contained in:
parent
888f42494e
commit
0e097874f8
15 changed files with 345 additions and 239 deletions
|
@ -12,7 +12,7 @@ use itertools::{Either, Itertools};
|
|||
use pep508_rs::Requirement;
|
||||
use platform_host::Platform;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_client::RegistryClientBuilder;
|
||||
use puffin_installer::{CachedDistribution, Downloader, LocalIndex, RemoteDistribution, Unzipper};
|
||||
use puffin_interpreter::PythonExecutable;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
@ -453,7 +453,7 @@ async fn resolve_and_install(
|
|||
}
|
||||
});
|
||||
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
let client = RegistryClientBuilder::default().cache(cache).build();
|
||||
|
||||
let platform = Platform::current()?;
|
||||
let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?;
|
||||
|
|
|
@ -13,11 +13,12 @@ use tracing::debug;
|
|||
use pep508_rs::Requirement;
|
||||
use platform_host::Platform;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_client::RegistryClientBuilder;
|
||||
use puffin_interpreter::PythonExecutable;
|
||||
use puffin_resolver::ResolutionMode;
|
||||
|
||||
use crate::commands::{elapsed, ExitStatus};
|
||||
use crate::index_urls::IndexUrls;
|
||||
use crate::printer::Printer;
|
||||
use crate::requirements::RequirementsSource;
|
||||
|
||||
|
@ -29,6 +30,7 @@ pub(crate) async fn pip_compile(
|
|||
constraints: &[RequirementsSource],
|
||||
output_file: Option<&Path>,
|
||||
mode: ResolutionMode,
|
||||
index_urls: Option<IndexUrls>,
|
||||
cache: Option<&Path>,
|
||||
mut printer: Printer,
|
||||
) -> Result<ExitStatus> {
|
||||
|
@ -61,7 +63,19 @@ pub(crate) async fn pip_compile(
|
|||
let tags = Tags::from_env(python.platform(), python.simple_version())?;
|
||||
|
||||
// Instantiate a client.
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
let client = {
|
||||
let mut builder = RegistryClientBuilder::default();
|
||||
builder = builder.cache(cache);
|
||||
if let Some(IndexUrls { index, extra_index }) = index_urls {
|
||||
if let Some(index) = index {
|
||||
builder = builder.index(index);
|
||||
}
|
||||
builder = builder.extra_index(extra_index);
|
||||
} else {
|
||||
builder = builder.no_index();
|
||||
}
|
||||
builder.build()
|
||||
};
|
||||
|
||||
// Resolve the dependencies.
|
||||
let resolver =
|
||||
|
|
|
@ -10,7 +10,7 @@ use tracing::debug;
|
|||
use pep508_rs::Requirement;
|
||||
use platform_host::Platform;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_client::RegistryClientBuilder;
|
||||
use puffin_installer::{
|
||||
CachedDistribution, Distribution, InstalledDistribution, LocalIndex, RemoteDistribution,
|
||||
SitePackages,
|
||||
|
@ -22,6 +22,7 @@ use crate::commands::reporters::{
|
|||
DownloadReporter, InstallReporter, UnzipReporter, WheelFinderReporter,
|
||||
};
|
||||
use crate::commands::{elapsed, ExitStatus};
|
||||
use crate::index_urls::IndexUrls;
|
||||
use crate::printer::Printer;
|
||||
use crate::requirements::RequirementsSource;
|
||||
|
||||
|
@ -29,6 +30,7 @@ use crate::requirements::RequirementsSource;
|
|||
pub(crate) async fn pip_sync(
|
||||
sources: &[RequirementsSource],
|
||||
link_mode: LinkMode,
|
||||
index_urls: Option<IndexUrls>,
|
||||
cache: Option<&Path>,
|
||||
mut printer: Printer,
|
||||
) -> Result<ExitStatus> {
|
||||
|
@ -44,13 +46,14 @@ pub(crate) async fn pip_sync(
|
|||
return Ok(ExitStatus::Success);
|
||||
}
|
||||
|
||||
sync_requirements(&requirements, link_mode, cache, printer).await
|
||||
sync_requirements(&requirements, link_mode, index_urls, cache, printer).await
|
||||
}
|
||||
|
||||
/// Install a set of locked requirements into the current Python environment.
|
||||
pub(crate) async fn sync_requirements(
|
||||
requirements: &[Requirement],
|
||||
link_mode: LinkMode,
|
||||
index_urls: Option<IndexUrls>,
|
||||
cache: Option<&Path>,
|
||||
mut printer: Printer,
|
||||
) -> Result<ExitStatus> {
|
||||
|
@ -92,7 +95,21 @@ pub(crate) async fn sync_requirements(
|
|||
|
||||
// Determine the current environment markers.
|
||||
let tags = Tags::from_env(python.platform(), python.simple_version())?;
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
|
||||
// Instantiate a client.
|
||||
let client = {
|
||||
let mut builder = RegistryClientBuilder::default();
|
||||
builder = builder.cache(cache);
|
||||
if let Some(IndexUrls { index, extra_index }) = index_urls {
|
||||
if let Some(index) = index {
|
||||
builder = builder.index(index);
|
||||
}
|
||||
builder = builder.extra_index(extra_index);
|
||||
} else {
|
||||
builder = builder.no_index();
|
||||
}
|
||||
builder.build()
|
||||
};
|
||||
|
||||
// Resolve the dependencies.
|
||||
let remote = if remote.is_empty() {
|
||||
|
|
19
crates/puffin-cli/src/index_urls.rs
Normal file
19
crates/puffin-cli/src/index_urls.rs
Normal file
|
@ -0,0 +1,19 @@
|
|||
use url::Url;
|
||||
|
||||
/// The index URLs to use for fetching packages.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct IndexUrls {
|
||||
pub(crate) index: Option<Url>,
|
||||
pub(crate) extra_index: Vec<Url>,
|
||||
}
|
||||
|
||||
impl IndexUrls {
|
||||
/// Determine the index URLs to use for fetching packages.
|
||||
pub(crate) fn from_args(
|
||||
index: Option<Url>,
|
||||
extra_index: Vec<Url>,
|
||||
no_index: bool,
|
||||
) -> Option<Self> {
|
||||
(!no_index).then_some(Self { index, extra_index })
|
||||
}
|
||||
}
|
|
@ -5,11 +5,14 @@ use clap::{Args, Parser, Subcommand};
|
|||
use colored::Colorize;
|
||||
use directories::ProjectDirs;
|
||||
use puffin_resolver::ResolutionMode;
|
||||
use url::Url;
|
||||
|
||||
use crate::commands::ExitStatus;
|
||||
use crate::index_urls::IndexUrls;
|
||||
use crate::requirements::RequirementsSource;
|
||||
|
||||
mod commands;
|
||||
mod index_urls;
|
||||
mod logging;
|
||||
mod printer;
|
||||
mod requirements;
|
||||
|
@ -74,6 +77,18 @@ struct PipCompileArgs {
|
|||
/// Write the compiled requirements to the given `requirements.txt` file.
|
||||
#[clap(short, long)]
|
||||
output_file: Option<PathBuf>,
|
||||
|
||||
/// The URL of the Python Package Index (default: https://pypi.org/simple).
|
||||
#[clap(long, short)]
|
||||
index_url: Option<Url>,
|
||||
|
||||
/// Extra URLs of package indexes to use, in addition to `--index-url`.
|
||||
#[clap(long)]
|
||||
extra_index_url: Vec<Url>,
|
||||
|
||||
/// Ignore the package index, instead relying on local archives and caches.
|
||||
#[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")]
|
||||
no_index: bool,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
|
@ -85,6 +100,18 @@ struct PipSyncArgs {
|
|||
/// The method to use when installing packages from the global cache.
|
||||
#[clap(long, value_enum)]
|
||||
link_mode: Option<install_wheel_rs::linker::LinkMode>,
|
||||
|
||||
/// The URL of the Python Package Index (default: https://pypi.org/simple).
|
||||
#[clap(long, short)]
|
||||
index_url: Option<Url>,
|
||||
|
||||
/// Extra URLs of package indexes to use, in addition to `--index-url`.
|
||||
#[clap(long)]
|
||||
extra_index_url: Vec<Url>,
|
||||
|
||||
/// Ignore the package index, instead relying on local archives and caches.
|
||||
#[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")]
|
||||
no_index: bool,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
|
@ -162,17 +189,22 @@ async fn main() -> ExitCode {
|
|||
.into_iter()
|
||||
.map(RequirementsSource::from)
|
||||
.collect::<Vec<_>>();
|
||||
let index_urls =
|
||||
IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index);
|
||||
commands::pip_compile(
|
||||
&requirements,
|
||||
&constraints,
|
||||
args.output_file.as_deref(),
|
||||
args.resolution.unwrap_or_default(),
|
||||
index_urls,
|
||||
cache_dir,
|
||||
printer,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Commands::PipSync(args) => {
|
||||
let index_urls =
|
||||
IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index);
|
||||
let sources = args
|
||||
.src_file
|
||||
.into_iter()
|
||||
|
@ -181,6 +213,7 @@ async fn main() -> ExitCode {
|
|||
commands::pip_sync(
|
||||
&sources,
|
||||
args.link_mode.unwrap_or_default(),
|
||||
index_urls,
|
||||
cache_dir,
|
||||
printer,
|
||||
)
|
||||
|
|
|
@ -1,186 +0,0 @@
|
|||
use std::fmt::Debug;
|
||||
|
||||
use futures::{AsyncRead, StreamExt, TryStreamExt};
|
||||
use reqwest::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::trace;
|
||||
use url::Url;
|
||||
|
||||
use puffin_package::metadata::Metadata21;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
||||
use crate::client::PypiClient;
|
||||
use crate::error::PypiClientError;
|
||||
|
||||
impl PypiClient {
|
||||
/// Fetch a package from the `PyPI` simple API.
|
||||
pub async fn simple(
|
||||
&self,
|
||||
package_name: impl AsRef<str>,
|
||||
) -> Result<SimpleJson, PypiClientError> {
|
||||
// Format the URL for PyPI.
|
||||
let mut url = self.registry.join("simple")?;
|
||||
url.path_segments_mut()
|
||||
.unwrap()
|
||||
.push(PackageName::normalize(&package_name).as_ref());
|
||||
url.path_segments_mut().unwrap().push("");
|
||||
url.set_query(Some("format=application/vnd.pypi.simple.v1+json"));
|
||||
|
||||
trace!(
|
||||
"Fetching metadata for {} from {}",
|
||||
package_name.as_ref(),
|
||||
url
|
||||
);
|
||||
|
||||
// Fetch from the registry.
|
||||
let text = self.simple_impl(&package_name, &url).await?;
|
||||
serde_json::from_str(&text)
|
||||
.map_err(move |e| PypiClientError::from_json_err(e, String::new()))
|
||||
}
|
||||
|
||||
async fn simple_impl(
|
||||
&self,
|
||||
package_name: impl AsRef<str>,
|
||||
url: &Url,
|
||||
) -> Result<String, PypiClientError> {
|
||||
Ok(self
|
||||
.client
|
||||
.get(url.clone())
|
||||
.header("Accept-Encoding", "gzip")
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()
|
||||
.map_err(|err| {
|
||||
if err.status() == Some(StatusCode::NOT_FOUND) {
|
||||
PypiClientError::PackageNotFound(
|
||||
(*self.registry).clone(),
|
||||
package_name.as_ref().to_string(),
|
||||
)
|
||||
} else {
|
||||
PypiClientError::RequestError(err)
|
||||
}
|
||||
})?
|
||||
.text()
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Fetch the metadata from a wheel file.
|
||||
pub async fn file(&self, file: File) -> Result<Metadata21, PypiClientError> {
|
||||
// Per PEP 658, if `data-dist-info-metadata` is available, we can request it directly;
|
||||
// otherwise, send to our dedicated caching proxy.
|
||||
let url = if file.data_dist_info_metadata.is_available() {
|
||||
Url::parse(&format!("{}.metadata", file.url))?
|
||||
} else {
|
||||
self.proxy.join(file.url.parse::<Url>()?.path())?
|
||||
};
|
||||
|
||||
trace!("Fetching file {} from {}", file.filename, url);
|
||||
|
||||
// Fetch from the registry.
|
||||
let text = self.file_impl(&file.filename, &url).await?;
|
||||
Metadata21::parse(text.as_bytes()).map_err(std::convert::Into::into)
|
||||
}
|
||||
|
||||
async fn file_impl(
|
||||
&self,
|
||||
filename: impl AsRef<str>,
|
||||
url: &Url,
|
||||
) -> Result<String, PypiClientError> {
|
||||
Ok(self
|
||||
.client
|
||||
.get(url.clone())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()
|
||||
.map_err(|err| {
|
||||
if err.status() == Some(StatusCode::NOT_FOUND) {
|
||||
PypiClientError::FileNotFound(
|
||||
(*self.registry).clone(),
|
||||
filename.as_ref().to_string(),
|
||||
)
|
||||
} else {
|
||||
PypiClientError::RequestError(err)
|
||||
}
|
||||
})?
|
||||
.text()
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Stream a file from an external URL.
|
||||
pub async fn stream_external(
|
||||
&self,
|
||||
url: &Url,
|
||||
) -> Result<Box<dyn AsyncRead + Unpin + Send + Sync>, PypiClientError> {
|
||||
Ok(Box::new(
|
||||
self.uncached_client
|
||||
.get(url.to_string())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.bytes_stream()
|
||||
.map(|r| match r {
|
||||
Ok(bytes) => Ok(bytes),
|
||||
Err(err) => Err(std::io::Error::new(std::io::ErrorKind::Other, err)),
|
||||
})
|
||||
.into_async_read(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SimpleJson {
|
||||
pub files: Vec<File>,
|
||||
pub meta: Meta,
|
||||
pub name: String,
|
||||
pub versions: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct File {
|
||||
pub core_metadata: Metadata,
|
||||
pub data_dist_info_metadata: Metadata,
|
||||
pub filename: String,
|
||||
pub hashes: Hashes,
|
||||
pub requires_python: Option<String>,
|
||||
pub size: usize,
|
||||
pub upload_time: String,
|
||||
pub url: String,
|
||||
pub yanked: Yanked,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum Metadata {
|
||||
Bool(bool),
|
||||
Hashes(Hashes),
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub fn is_available(&self) -> bool {
|
||||
match self {
|
||||
Self::Bool(is_available) => *is_available,
|
||||
Self::Hashes(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum Yanked {
|
||||
Bool(bool),
|
||||
Reason(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Hashes {
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct Meta {
|
||||
#[serde(rename = "_last-serial")]
|
||||
pub last_serial: i64,
|
||||
pub api_version: String,
|
||||
}
|
|
@ -1,25 +1,39 @@
|
|||
use std::fmt::Debug;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{AsyncRead, StreamExt, TryStreamExt};
|
||||
use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions};
|
||||
use reqwest::ClientBuilder;
|
||||
use reqwest::StatusCode;
|
||||
use reqwest_middleware::ClientWithMiddleware;
|
||||
use reqwest_retry::policies::ExponentialBackoff;
|
||||
use reqwest_retry::RetryTransientMiddleware;
|
||||
use tracing::trace;
|
||||
use url::Url;
|
||||
|
||||
use puffin_package::metadata::Metadata21;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::types::{File, SimpleJson};
|
||||
|
||||
/// A builder for an [`RegistryClient`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PypiClientBuilder {
|
||||
registry: Url,
|
||||
pub struct RegistryClientBuilder {
|
||||
index: Url,
|
||||
extra_index: Vec<Url>,
|
||||
no_index: bool,
|
||||
proxy: Url,
|
||||
retries: u32,
|
||||
cache: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Default for PypiClientBuilder {
|
||||
impl Default for RegistryClientBuilder {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
registry: Url::parse("https://pypi.org").unwrap(),
|
||||
index: Url::parse("https://pypi.org/simple").unwrap(),
|
||||
extra_index: vec![],
|
||||
no_index: false,
|
||||
proxy: Url::parse("https://pypi-metadata.ruff.rs").unwrap(),
|
||||
cache: None,
|
||||
retries: 0,
|
||||
|
@ -27,10 +41,22 @@ impl Default for PypiClientBuilder {
|
|||
}
|
||||
}
|
||||
|
||||
impl PypiClientBuilder {
|
||||
impl RegistryClientBuilder {
|
||||
#[must_use]
|
||||
pub fn registry(mut self, registry: Url) -> Self {
|
||||
self.registry = registry;
|
||||
pub fn index(mut self, index: Url) -> Self {
|
||||
self.index = index;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn extra_index(mut self, extra_index: Vec<Url>) -> Self {
|
||||
self.extra_index = extra_index;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn no_index(mut self) -> Self {
|
||||
self.no_index = true;
|
||||
self
|
||||
}
|
||||
|
||||
|
@ -55,7 +81,7 @@ impl PypiClientBuilder {
|
|||
self
|
||||
}
|
||||
|
||||
pub fn build(self) -> PypiClient {
|
||||
pub fn build(self) -> RegistryClient {
|
||||
let client_raw = {
|
||||
let client_core = ClientBuilder::new()
|
||||
.user_agent("puffin")
|
||||
|
@ -85,19 +111,139 @@ impl PypiClientBuilder {
|
|||
let uncached_client_builder =
|
||||
reqwest_middleware::ClientBuilder::new(client_raw).with(retry_strategy);
|
||||
|
||||
PypiClient {
|
||||
registry: Arc::new(self.registry),
|
||||
proxy: Arc::new(self.proxy),
|
||||
RegistryClient {
|
||||
index: self.index,
|
||||
extra_index: self.extra_index,
|
||||
no_index: self.no_index,
|
||||
proxy: self.proxy,
|
||||
client: client_builder.build(),
|
||||
uncached_client: uncached_client_builder.build(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A client for fetching packages from a `PyPI`-compatible index.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PypiClient {
|
||||
pub(crate) registry: Arc<Url>,
|
||||
pub(crate) proxy: Arc<Url>,
|
||||
pub struct RegistryClient {
|
||||
pub(crate) index: Url,
|
||||
pub(crate) extra_index: Vec<Url>,
|
||||
pub(crate) no_index: bool,
|
||||
pub(crate) proxy: Url,
|
||||
pub(crate) client: ClientWithMiddleware,
|
||||
pub(crate) uncached_client: ClientWithMiddleware,
|
||||
}
|
||||
|
||||
impl RegistryClient {
|
||||
/// Fetch a package from the `PyPI` simple API.
|
||||
pub async fn simple(&self, package_name: impl AsRef<str>) -> Result<SimpleJson, Error> {
|
||||
if self.no_index {
|
||||
return Err(Error::PackageNotFound(package_name.as_ref().to_string()));
|
||||
}
|
||||
|
||||
for index in std::iter::once(&self.index).chain(self.extra_index.iter()) {
|
||||
// Format the URL for PyPI.
|
||||
let mut url = index.clone();
|
||||
url.path_segments_mut()
|
||||
.unwrap()
|
||||
.push(PackageName::normalize(&package_name).as_ref());
|
||||
url.path_segments_mut().unwrap().push("");
|
||||
url.set_query(Some("format=application/vnd.pypi.simple.v1+json"));
|
||||
|
||||
trace!(
|
||||
"Fetching metadata for {} from {}",
|
||||
package_name.as_ref(),
|
||||
url
|
||||
);
|
||||
|
||||
// Fetch from the index.
|
||||
match self.simple_impl(&url).await {
|
||||
Ok(text) => {
|
||||
return serde_json::from_str(&text)
|
||||
.map_err(move |e| Error::from_json_err(e, String::new()));
|
||||
}
|
||||
Err(err) => {
|
||||
if err.status() == Some(StatusCode::NOT_FOUND) {
|
||||
continue;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::PackageNotFound(package_name.as_ref().to_string()))
|
||||
}
|
||||
|
||||
async fn simple_impl(&self, url: &Url) -> Result<String, reqwest_middleware::Error> {
|
||||
Ok(self
|
||||
.client
|
||||
.get(url.clone())
|
||||
.header("Accept-Encoding", "gzip")
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Fetch the metadata from a wheel file.
|
||||
pub async fn file(&self, file: File) -> Result<Metadata21, Error> {
|
||||
if self.no_index {
|
||||
return Err(Error::FileNotFound(file.filename));
|
||||
}
|
||||
|
||||
// Per PEP 658, if `data-dist-info-metadata` is available, we can request it directly;
|
||||
// otherwise, send to our dedicated caching proxy.
|
||||
let url = if file.data_dist_info_metadata.is_available() {
|
||||
Url::parse(&format!("{}.metadata", file.url))?
|
||||
} else {
|
||||
self.proxy.join(file.url.parse::<Url>()?.path())?
|
||||
};
|
||||
|
||||
trace!("Fetching file {} from {}", file.filename, url);
|
||||
|
||||
// Fetch from the index.
|
||||
let text = self.file_impl(&url).await.map_err(|err| {
|
||||
if err.status() == Some(StatusCode::NOT_FOUND) {
|
||||
Error::FileNotFound(file.filename.to_string())
|
||||
} else {
|
||||
err.into()
|
||||
}
|
||||
})?;
|
||||
Metadata21::parse(text.as_bytes()).map_err(std::convert::Into::into)
|
||||
}
|
||||
|
||||
async fn file_impl(&self, url: &Url) -> Result<String, reqwest_middleware::Error> {
|
||||
Ok(self
|
||||
.client
|
||||
.get(url.clone())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Stream a file from an external URL.
|
||||
pub async fn stream_external(
|
||||
&self,
|
||||
url: &Url,
|
||||
) -> Result<Box<dyn AsyncRead + Unpin + Send + Sync>, Error> {
|
||||
if self.no_index {
|
||||
return Err(Error::ResourceNotFound(url.clone()));
|
||||
}
|
||||
|
||||
Ok(Box::new(
|
||||
self.uncached_client
|
||||
.get(url.to_string())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.bytes_stream()
|
||||
.map(|r| match r {
|
||||
Ok(bytes) => Ok(bytes),
|
||||
Err(err) => Err(std::io::Error::new(std::io::ErrorKind::Other, err)),
|
||||
})
|
||||
.into_async_read(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ use url::Url;
|
|||
use puffin_package::metadata;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum PypiClientError {
|
||||
pub enum Error {
|
||||
/// An invalid URL was provided.
|
||||
#[error(transparent)]
|
||||
UrlParseError(#[from] url::ParseError),
|
||||
|
@ -13,16 +13,20 @@ pub enum PypiClientError {
|
|||
///
|
||||
/// Make sure the package name is spelled correctly and that you've
|
||||
/// configured the right registry to fetch it from.
|
||||
#[error("Package `{1}` was not found in registry {0}.")]
|
||||
PackageNotFound(Url, String),
|
||||
#[error("Package `{0}` was not found in the registry.")]
|
||||
PackageNotFound(String),
|
||||
|
||||
/// The metadata file could not be parsed.
|
||||
#[error(transparent)]
|
||||
MetadataParseError(#[from] metadata::Error),
|
||||
|
||||
/// The metadata file was not found in the registry.
|
||||
#[error("File `{1}` was not found in registry {0}.")]
|
||||
FileNotFound(Url, String),
|
||||
#[error("File `{0}` was not found in the registry.")]
|
||||
FileNotFound(String),
|
||||
|
||||
/// The resource was not found in the registry.
|
||||
#[error("Resource `{0}` was not found in the registry.")]
|
||||
ResourceNotFound(Url),
|
||||
|
||||
/// A generic request error happened while making a request. Refer to the
|
||||
/// error message for more details.
|
||||
|
@ -41,7 +45,7 @@ pub enum PypiClientError {
|
|||
},
|
||||
}
|
||||
|
||||
impl PypiClientError {
|
||||
impl Error {
|
||||
pub fn from_json_err(err: serde_json::Error, url: String) -> Self {
|
||||
Self::BadJson { source: err, url }
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
pub use api::{File, SimpleJson};
|
||||
pub use client::{PypiClient, PypiClientBuilder};
|
||||
pub use error::PypiClientError;
|
||||
pub use client::{RegistryClient, RegistryClientBuilder};
|
||||
pub use error::Error;
|
||||
pub use types::{File, SimpleJson};
|
||||
|
||||
mod api;
|
||||
mod client;
|
||||
mod error;
|
||||
mod types;
|
||||
|
|
59
crates/puffin-client/src/types.rs
Normal file
59
crates/puffin-client/src/types.rs
Normal file
|
@ -0,0 +1,59 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SimpleJson {
|
||||
pub files: Vec<File>,
|
||||
pub meta: Meta,
|
||||
pub name: String,
|
||||
pub versions: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct File {
|
||||
pub core_metadata: Metadata,
|
||||
pub data_dist_info_metadata: Metadata,
|
||||
pub filename: String,
|
||||
pub hashes: Hashes,
|
||||
pub requires_python: Option<String>,
|
||||
pub size: usize,
|
||||
pub upload_time: String,
|
||||
pub url: String,
|
||||
pub yanked: Yanked,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum Metadata {
|
||||
Bool(bool),
|
||||
Hashes(Hashes),
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub fn is_available(&self) -> bool {
|
||||
match self {
|
||||
Self::Bool(is_available) => *is_available,
|
||||
Self::Hashes(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum Yanked {
|
||||
Bool(bool),
|
||||
Reason(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Hashes {
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct Meta {
|
||||
#[serde(rename = "_last-serial")]
|
||||
pub last_serial: i64,
|
||||
pub api_version: String,
|
||||
}
|
|
@ -8,21 +8,21 @@ use tracing::debug;
|
|||
use url::Url;
|
||||
|
||||
use pep440_rs::Version;
|
||||
use puffin_client::PypiClient;
|
||||
use puffin_client::RegistryClient;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
||||
use crate::cache::WheelCache;
|
||||
use crate::distribution::RemoteDistribution;
|
||||
|
||||
pub struct Downloader<'a> {
|
||||
client: &'a PypiClient,
|
||||
client: &'a RegistryClient,
|
||||
cache: Option<&'a Path>,
|
||||
reporter: Option<Box<dyn Reporter>>,
|
||||
}
|
||||
|
||||
impl<'a> Downloader<'a> {
|
||||
/// Initialize a new downloader.
|
||||
pub fn new(client: &'a PypiClient, cache: Option<&'a Path>) -> Self {
|
||||
pub fn new(client: &'a RegistryClient, cache: Option<&'a Path>) -> Self {
|
||||
Self {
|
||||
client,
|
||||
cache,
|
||||
|
@ -91,7 +91,7 @@ pub struct InMemoryDistribution {
|
|||
/// Download a wheel to a given path.
|
||||
async fn fetch_wheel(
|
||||
remote: RemoteDistribution,
|
||||
client: PypiClient,
|
||||
client: RegistryClient,
|
||||
cache: Option<impl AsRef<Path>>,
|
||||
) -> Result<InMemoryDistribution> {
|
||||
// Parse the wheel's SRI.
|
||||
|
|
|
@ -15,7 +15,7 @@ pub enum ResolveError {
|
|||
StreamTermination,
|
||||
|
||||
#[error(transparent)]
|
||||
Client(#[from] puffin_client::PypiClientError),
|
||||
Client(#[from] puffin_client::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
TrySend(#[from] futures::channel::mpsc::SendError),
|
||||
|
|
|
@ -21,7 +21,7 @@ use waitmap::WaitMap;
|
|||
use distribution_filename::WheelFilename;
|
||||
use pep508_rs::{MarkerEnvironment, Requirement};
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::{File, PypiClient, SimpleJson};
|
||||
use puffin_client::{File, RegistryClient, SimpleJson};
|
||||
use puffin_package::dist_info_name::DistInfoName;
|
||||
use puffin_package::metadata::Metadata21;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
@ -38,7 +38,7 @@ pub struct Resolver<'a> {
|
|||
constraints: Vec<Requirement>,
|
||||
markers: &'a MarkerEnvironment,
|
||||
tags: &'a Tags,
|
||||
client: &'a PypiClient,
|
||||
client: &'a RegistryClient,
|
||||
selector: CandidateSelector,
|
||||
cache: Arc<SolverCache>,
|
||||
}
|
||||
|
@ -51,7 +51,7 @@ impl<'a> Resolver<'a> {
|
|||
mode: ResolutionMode,
|
||||
markers: &'a MarkerEnvironment,
|
||||
tags: &'a Tags,
|
||||
client: &'a PypiClient,
|
||||
client: &'a RegistryClient,
|
||||
) -> Self {
|
||||
Self {
|
||||
selector: CandidateSelector::from_mode(mode, &requirements),
|
||||
|
|
|
@ -14,7 +14,7 @@ use tracing::debug;
|
|||
use distribution_filename::WheelFilename;
|
||||
use pep508_rs::Requirement;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::{File, PypiClient, SimpleJson};
|
||||
use puffin_client::{File, RegistryClient, SimpleJson};
|
||||
use puffin_package::metadata::Metadata21;
|
||||
use puffin_package::package_name::PackageName;
|
||||
|
||||
|
@ -23,13 +23,13 @@ use crate::resolution::{PinnedPackage, Resolution};
|
|||
|
||||
pub struct WheelFinder<'a> {
|
||||
tags: &'a Tags,
|
||||
client: &'a PypiClient,
|
||||
client: &'a RegistryClient,
|
||||
reporter: Option<Box<dyn Reporter>>,
|
||||
}
|
||||
|
||||
impl<'a> WheelFinder<'a> {
|
||||
/// Initialize a new wheel finder.
|
||||
pub fn new(tags: &'a Tags, client: &'a PypiClient) -> Self {
|
||||
pub fn new(tags: &'a Tags, client: &'a RegistryClient) -> Self {
|
||||
Self {
|
||||
tags,
|
||||
client,
|
||||
|
|
|
@ -11,14 +11,14 @@ use once_cell::sync::Lazy;
|
|||
use pep508_rs::{MarkerEnvironment, Requirement, StringVersion};
|
||||
use platform_host::{Arch, Os, Platform};
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_client::RegistryClientBuilder;
|
||||
use puffin_resolver::{ResolutionMode, Resolver};
|
||||
|
||||
#[tokio::test]
|
||||
async fn pylint() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("pylint==2.3.0").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
@ -41,7 +41,7 @@ async fn pylint() -> Result<()> {
|
|||
async fn black() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
@ -64,7 +64,7 @@ async fn black() -> Result<()> {
|
|||
async fn black_colorama() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black[colorama]<=23.9.1").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
@ -87,7 +87,7 @@ async fn black_colorama() -> Result<()> {
|
|||
async fn black_python_310() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
@ -112,7 +112,7 @@ async fn black_python_310() -> Result<()> {
|
|||
async fn black_mypy_extensions() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
|
||||
let constraints = vec![Requirement::from_str("mypy-extensions<1").unwrap()];
|
||||
|
@ -137,7 +137,7 @@ async fn black_mypy_extensions() -> Result<()> {
|
|||
async fn black_mypy_extensions_extra() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
|
||||
let constraints = vec![Requirement::from_str("mypy-extensions[extra]<1").unwrap()];
|
||||
|
@ -162,7 +162,7 @@ async fn black_mypy_extensions_extra() -> Result<()> {
|
|||
async fn black_flake8() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()];
|
||||
let constraints = vec![Requirement::from_str("flake8<1").unwrap()];
|
||||
|
@ -185,7 +185,7 @@ async fn black_flake8() -> Result<()> {
|
|||
async fn black_lowest() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black>21").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
@ -208,7 +208,7 @@ async fn black_lowest() -> Result<()> {
|
|||
async fn black_lowest_direct() -> Result<()> {
|
||||
colored::control::set_override(false);
|
||||
|
||||
let client = PypiClientBuilder::default().build();
|
||||
let client = RegistryClientBuilder::default().build();
|
||||
|
||||
let requirements = vec![Requirement::from_str("black>21").unwrap()];
|
||||
let constraints = vec![];
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue