mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-11-03 21:23:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add support for local directories with --index-url (#4226)

Browse source 
## Summary

Closes #4078.
This commit is contained in:
Charlie Marsh 2024-06-10 19:27:04 -07:00 • committed by GitHub
parent f296ef08d6
commit 656fc427b9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 219 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/distribution-types/src/file.rs
View file

@ -2,7 +2,6 @@ use std::fmt::{Display, Formatter};
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use url::Url;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
@ -10,7 +9,7 @@ use pep508_rs::split_scheme;
use pypi_types::{CoreMetadata, HashDigest, Yanked};
/// Error converting [`pypi_types::File`] to [`distribution_type::File`].
#[derive(Debug, Error)]
#[derive(Debug, thiserror::Error)]
pub enum FileConversionError {
#[error("Failed to parse 'requires-python': `{0}`")]
RequiresPython(String, #[source] VersionSpecifiersParseError),
@ -57,12 +56,10 @@ impl File {
.map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?,
size: file.size,
upload_time_utc_ms: file.upload_time.map(|dt| dt.timestamp_millis()),
url: {
if split_scheme(&file.url).is_some() {
FileLocation::AbsoluteUrl(file.url)
} else {
FileLocation::RelativeUrl(base.to_string(), file.url)
}
url: if split_scheme(&file.url).is_some() {
FileLocation::AbsoluteUrl(file.url)
} else {
FileLocation::RelativeUrl(base.to_string(), file.url)
},
yanked: file.yanked,
})

2
crates/distribution-types/src/index_url.rs
View file

@ -98,6 +98,8 @@ impl FromStr for IndexUrl {
let url = VerbatimUrl::from_url(url).with_given(s.to_owned());
if *url.raw() == *PYPI_URL {
Ok(Self::Pypi(url))
} else if url.scheme() == "file" {
Ok(Self::Path(url))
} else {
Ok(Self::Url(url))
}

1
crates/pep508-rs/src/verbatim_url.rs
View file

@ -235,6 +235,7 @@ impl Pep508Url for VerbatimUrl {
.with_given(url.to_string()),
)
}
// Ex) `https://download.pytorch.org/whl/torch_stable.html`
Some(_) => {
// Ex) `https://download.pytorch.org/whl/torch_stable.html`

10
crates/uv-client/src/error.rs
View file

@ -128,15 +128,15 @@ impl From<ErrorKind> for Error {
#[derive(Debug, thiserror::Error)]
pub enum ErrorKind {
/// An invalid URL was provided.
#[error(transparent)]
UrlParseError(#[from] url::ParseError),
UrlParse(#[from] url::ParseError),
/// A base URL could not be joined with a possibly relative URL.
#[error(transparent)]
JoinRelativeError(#[from] pypi_types::JoinRelativeError),
JoinRelativeUrl(#[from] pypi_types::JoinRelativeError),
#[error("Expected a file URL, but received: {0}")]
NonFileUrl(Url),
/// Dist-info error
#[error(transparent)]
DistInfo(#[from] install_wheel_rs::Error),

109
crates/uv-client/src/registry_client.rs
View file

@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use std::fmt::Debug;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use async_http_range_reader::AsyncHttpRangeReader;
@ -20,7 +20,7 @@ use pep440_rs::Version;
use pep508_rs::MarkerEnvironment;
use platform_tags::Platform;
use pypi_types::{Metadata23, SimpleJson};
use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache};
use uv_configuration::IndexStrategy;
use uv_configuration::KeyringProviderType;
use uv_normalize::PackageName;
@ -258,6 +258,10 @@ impl RegistryClient {
Ok(results)
}
/// Fetch the [`SimpleMetadata`] from a single index for a given package.
///
/// The index can either be a PEP 503-compatible remote repository, or a local directory laid
/// out in the same format.
async fn simple_single_index(
&self,
package_name: &PackageName,
@ -293,6 +297,22 @@ impl RegistryClient {
Connectivity::Offline => CacheControl::AllowStale,
};
if matches!(index, IndexUrl::Path(_)) {
self.fetch_local_index(package_name, &url).await.map(Ok)
} else {
self.fetch_remote_index(package_name, &url, &cache_entry, cache_control)
.await
}
}
/// Fetch the [`SimpleMetadata`] from a remote URL, using the PEP 503 Simple Repository API.
async fn fetch_remote_index(
&self,
package_name: &PackageName,
url: &Url,
cache_entry: &CacheEntry,
cache_control: CacheControl,
) -> Result<Result<OwnedArchive<SimpleMetadata>, CachedClientError<Error>>, Error> {
let simple_request = self
.uncached_client()
.get(url.clone())
@ -331,10 +351,7 @@ impl RegistryClient {
}
MediaType::Html => {
let text = response.text().await.map_err(ErrorKind::from)?;
let SimpleHtml { base, files } = SimpleHtml::parse(&text, &url)
.map_err(|err| Error::from_html_err(err, url.clone()))?;
SimpleMetadata::from_files(files, package_name, base.as_url())
SimpleMetadata::from_html(&text, package_name, &url)?
}
};
OwnedArchive::from_unarchived(&unarchived)
@ -346,7 +363,7 @@ impl RegistryClient {
.cached_client()
.get_cacheable(
simple_request,
&cache_entry,
cache_entry,
cache_control,
parse_simple_response,
)
@ -354,6 +371,24 @@ impl RegistryClient {
Ok(result)
}
/// Fetch the [`SimpleMetadata`] from a local file, using a PEP 503-compatible directory
/// structure.
async fn fetch_local_index(
&self,
package_name: &PackageName,
url: &Url,
) -> Result<OwnedArchive<SimpleMetadata>, Error> {
let path = url
.to_file_path()
.map_err(|_| ErrorKind::NonFileUrl(url.clone()))?
.join("index.html");
let text = fs_err::tokio::read_to_string(&path)
.await
.map_err(ErrorKind::from)?;
let metadata = SimpleMetadata::from_html(&text, package_name, url)?;
OwnedArchive::from_unarchived(&metadata)
}
/// Fetch the metadata for a remote wheel file.
///
/// For a remote wheel, we try the following ways to fetch the metadata:
@ -364,20 +399,45 @@ impl RegistryClient {
pub async fn wheel_metadata(&self, built_dist: &BuiltDist) -> Result<Metadata23, Error> {
let metadata = match &built_dist {
BuiltDist::Registry(wheels) => {
#[derive(Debug, Clone)]
enum WheelLocation {
/// A local file path.
Path(PathBuf),
/// A remote URL.
Url(Url),
}
let wheel = wheels.best_wheel();
match &wheel.file.url {
let location = match &wheel.file.url {
FileLocation::RelativeUrl(base, url) => {
let url = pypi_types::base_url_join_relative(base, url)
.map_err(ErrorKind::JoinRelativeError)?;
self.wheel_metadata_registry(&wheel.index, &wheel.file, &url)
.await?
.map_err(ErrorKind::JoinRelativeUrl)?;
if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|_| ErrorKind::NonFileUrl(url.clone()))?;
WheelLocation::Path(path)
} else {
WheelLocation::Url(url)
}
}
FileLocation::AbsoluteUrl(url) => {
let url = Url::parse(url).map_err(ErrorKind::UrlParseError)?;
self.wheel_metadata_registry(&wheel.index, &wheel.file, &url)
.await?
let url = Url::parse(url).map_err(ErrorKind::UrlParse)?;
if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|_| ErrorKind::NonFileUrl(url.clone()))?;
WheelLocation::Path(path)
} else {
WheelLocation::Url(url)
}
}
FileLocation::Path(path) => {
FileLocation::Path(path) => WheelLocation::Path(path.clone()),
};
match location {
WheelLocation::Path(path) => {
let file = fs_err::tokio::File::open(&path)
.await
.map_err(ErrorKind::Io)?;
@ -385,6 +445,10 @@ impl RegistryClient {
read_metadata_async_seek(&wheel.filename, built_dist.to_string(), reader)
.await?
}
WheelLocation::Url(url) => {
self.wheel_metadata_registry(&wheel.index, &wheel.file, &url)
.await?
}
}
}
BuiltDist::DirectUrl(wheel) => {
@ -599,7 +663,7 @@ impl RegistryClient {
std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!(
"Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: {}s).", self.timeout()
"Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: {}s).", self.timeout()
),
)
} else {
@ -772,7 +836,6 @@ impl SimpleMetadata {
DistFilename::SourceDistFilename(ref inner) => &inner.version,
DistFilename::WheelFilename(ref inner) => &inner.version,
};
let file = match File::try_from(file, base) {
Ok(file) => file,
Err(err) => {
@ -799,6 +862,18 @@ impl SimpleMetadata {
.collect(),
)
}
/// Read the [`SimpleMetadata`] from an HTML index.
fn from_html(text: &str, package_name: &PackageName, url: &Url) -> Result<Self, Error> {
let SimpleHtml { base, files } =
SimpleHtml::parse(text, url).map_err(|err| Error::from_html_err(err, url.clone()))?;
Ok(SimpleMetadata::from_files(
files,
package_name,
base.as_url(),
))
}
}
impl IntoIterator for SimpleMetadata {

11
crates/uv-distribution/src/distribution_database.rs
View file

@ -171,7 +171,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
WheelCache::Index(&wheel.index).wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
return self
.load_wheel(path, &wheel.filename, cache_entry, dist, hashes)
.await;
@ -185,6 +184,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
wheel.filename.stem(),
);
// If the URL is a file URL, load the wheel directly.
if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|()| Error::NonFileUrl(url.clone()))?;
return self
.load_wheel(&path, &wheel.filename, wheel_entry, dist, hashes)
.await;
}
// Download and unzip.
match self
.stream_wheel(

3
crates/uv-distribution/src/error.rs
View file

@ -1,6 +1,7 @@
use std::path::PathBuf;
use tokio::task::JoinError;
use url::Url;
use zip::result::ZipError;
use crate::metadata::MetadataError;
@ -25,6 +26,8 @@ pub enum Error {
RelativePath(PathBuf),
#[error(transparent)]
JoinRelativeUrl(#[from] pypi_types::JoinRelativeError),
#[error("Expected a file URL, but received: {0}")]
NonFileUrl(Url),
#[error(transparent)]
Git(#[from] uv_git::GitResolverError),
#[error(transparent)]

39
crates/uv-distribution/src/source/mod.rs
View file

@ -124,6 +124,26 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
};
// If the URL is a file URL, use the local path directly.
if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|()| Error::NonFileUrl(url.clone()))?;
return self
.archive(
source,
&PathSourceUrl {
url: &url,
path: Cow::Owned(path),
},
&cache_shard,
tags,
hashes,
)
.boxed_local()
.await;
}
self.url(
source,
&dist.file.filename,
@ -281,6 +301,25 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
};
// If the URL is a file URL, use the local path directly.
if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|()| Error::NonFileUrl(url.clone()))?;
return self
.archive_metadata(
source,
&PathSourceUrl {
url: &url,
path: Cow::Owned(path),
},
&cache_shard,
hashes,
)
.boxed_local()
.await;
}
self.url_metadata(
source,
&dist.file.filename,

12
crates/uv/src/cli.rs
View file

@ -1655,6 +1655,9 @@ pub(crate) struct VenvArgs {
/// The URL of the Python package index (by default: <https://pypi.org/simple>).
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// The index given by this flag is given lower priority than all other
/// indexes specified via the `--extra-index-url` flag.
///
@ -1666,6 +1669,9 @@ pub(crate) struct VenvArgs {
/// Extra URLs of package indexes to use, in addition to `--index-url`.
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// All indexes given via this flag take priority over the index
/// in `--index-url` (which defaults to PyPI). And when multiple
/// `--extra-index-url` flags are given, earlier values take priority.
@ -2018,6 +2024,9 @@ pub(crate) struct ToolchainInstallArgs {
pub(crate) struct IndexArgs {
/// The URL of the Python package index (by default: <https://pypi.org/simple>).
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// The index given by this flag is given lower priority than all other
/// indexes specified via the `--extra-index-url` flag.
///
@ -2029,6 +2038,9 @@ pub(crate) struct IndexArgs {
/// Extra URLs of package indexes to use, in addition to `--index-url`.
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// All indexes given via this flag take priority over the index
/// in `--index-url` (which defaults to PyPI). And when multiple
/// `--extra-index-url` flags are given, earlier values take priority.

50
crates/uv/tests/pip_install.rs
View file

@ -8,6 +8,7 @@ use assert_fs::prelude::*;
use base64::{prelude::BASE64_STANDARD as base64, Engine};
use indoc::indoc;
use itertools::Itertools;
use url::Url;
use common::{uv_snapshot, TestContext};
use uv_fs::Simplified;
@ -5306,3 +5307,52 @@ fn prefer_editable() -> Result<()> {
Ok(())
}
/// Resolve against a local directory laid out as a PEP 503-compatible index.
#[test]
fn local_index() -> Result<()> {
let context = TestContext::new("3.12");
let root = context.temp_dir.child("simple-html");
fs_err::create_dir_all(&root)?;
let tqdm = root.child("tqdm");
fs_err::create_dir_all(&tqdm)?;
let index = tqdm.child("index.html");
index.write_str(&indoc::formatdoc! {r#"
<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for example-a-961b4c22</h1>
<a
href="{}/tqdm-1000.0.0-py3-none-any.whl"
data-requires-python=">=3.8"
>
tqdm-1000.0.0-py3-none-any.whl
</a>
</body>
</html>
"#, Url::from_directory_path(context.workspace_root.join("scripts/links/")).unwrap().as_str()})?;
uv_snapshot!(context.filters(), context.install_without_exclude_newer()
.arg("tqdm")
.arg("--index-url")
.arg(Url::from_directory_path(root).unwrap().as_str()), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
+ tqdm==1000.0.0
"###
);
Ok(())
}