mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Rename to uv (#1302)

Browse source 
First, replace all usages in files in-place. I used my editor for this.
If someone wants to add a one-liner that'd be fun.

Then, update directory and file names:

```
# Run twice for nested directories
find . -type d -print0 | xargs -0 rename s/puffin/uv/g
find . -type d -print0 | xargs -0 rename s/puffin/uv/g

# Update files
find . -type f -print0 | xargs -0 rename s/puffin/uv/g
```

Then add all the files again

```
# Add all the files again
git add crates
git add python/uv

# This one needs a force-add
git add -f crates/uv-trampoline
```
This commit is contained in:
Zanie Blue 2024-02-15 11:19:46 -06:00 • committed by GitHub
parent 328b116d5d
commit 2586f655bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
229 changed files with 1796 additions and 1818 deletions
Download patch file Download diff file Expand all files Collapse all files

46
crates/uv-distribution/Cargo.toml Normal file
View file

@ -0,0 +1,46 @@
[package]
name = "uv-distribution"
version = "0.0.1"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[lints]
workspace = true
[dependencies]
cache-key = { path = "../cache-key" }
distribution-filename = { path = "../distribution-filename", features = ["serde"] }
distribution-types = { path = "../distribution-types" }
install-wheel-rs = { path = "../install-wheel-rs" }
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
platform-tags = { path = "../platform-tags" }
uv-cache = { path = "../uv-cache" }
uv-client = { path = "../uv-client" }
uv-extract = { path = "../uv-extract" }
uv-fs = { path = "../uv-fs", features = ["tokio"] }
uv-git = { path = "../uv-git", features = ["vendored-openssl"] }
uv-normalize = { path = "../uv-normalize" }
uv-traits = { path = "../uv-traits" }
pypi-types = { path = "../pypi-types" }
anyhow = { workspace = true }
fs-err = { workspace = true }
futures = { workspace = true }
nanoid = { workspace = true }
reqwest = { workspace = true }
rmp-serde = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true , features = ["derive"] }
tempfile = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true, features = ["compat"] }
tracing = { workspace = true }
url = { workspace = true }
zip = { workspace = true }

442
crates/uv-distribution/src/distribution_database.rs Normal file
View file

@ -0,0 +1,442 @@
use std::borrow::Cow;
use std::io;
use std::path::Path;
use std::sync::Arc;
use futures::{FutureExt, TryStreamExt};
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info_span, instrument, Instrument};
use url::Url;
use distribution_types::{
BuiltDist, DirectGitUrl, Dist, FileLocation, IndexLocations, LocalEditable, Name, SourceDist,
};
use platform_tags::Tags;
use pypi_types::Metadata21;
use uv_cache::{Cache, CacheBucket, Timestamp, WheelCache};
use uv_client::{CacheControl, CachedClientError, Connectivity, RegistryClient};
use uv_fs::metadata_if_exists;
use uv_git::GitSource;
use uv_traits::{BuildContext, NoBinary, NoBuild};
use crate::download::{BuiltWheel, UnzippedWheel};
use crate::locks::Locks;
use crate::reporter::Facade;
use crate::{DiskWheel, Error, LocalWheel, Reporter, SourceDistCachedBuilder};
/// A cached high-level interface to convert distributions (a requirement resolved to a location)
/// to a wheel or wheel metadata.
///
/// For wheel metadata, this happens by either fetching the metadata from the remote wheel or by
/// building the source distribution. For wheel files, either the wheel is downloaded or a source
/// distribution is downloaded, built and the new wheel gets returned.
///
/// All kinds of wheel sources (index, url, path) and source distribution source (index, url, path,
/// git) are supported.
///
/// This struct also has the task of acquiring locks around source dist builds in general and git
/// operation especially.
pub struct DistributionDatabase<'a, Context: BuildContext + Send + Sync> {
cache: &'a Cache,
reporter: Option<Arc<dyn Reporter>>,
locks: Arc<Locks>,
client: &'a RegistryClient,
build_context: &'a Context,
builder: SourceDistCachedBuilder<'a, Context>,
}
impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> {
pub fn new(
cache: &'a Cache,
tags: &'a Tags,
client: &'a RegistryClient,
build_context: &'a Context,
) -> Self {
Self {
cache,
reporter: None,
locks: Arc::new(Locks::default()),
client,
build_context,
builder: SourceDistCachedBuilder::new(build_context, client, tags),
}
}
/// Set the [`Reporter`] to use for this source distribution fetcher.
#[must_use]
pub fn with_reporter(self, reporter: impl Reporter + 'static) -> Self {
let reporter = Arc::new(reporter);
Self {
reporter: Some(reporter.clone()),
builder: self.builder.with_reporter(reporter),
..self
}
}
/// Either fetch the wheel or fetch and build the source distribution
///
/// If `no_remote_wheel` is set, the wheel will be built from a source distribution
/// even if compatible pre-built wheels are available.
#[instrument(skip(self))]
pub async fn get_or_build_wheel(&self, dist: Dist) -> Result<LocalWheel, Error> {
let no_binary = match self.build_context.no_binary() {
NoBinary::None => false,
NoBinary::All => true,
NoBinary::Packages(packages) => packages.contains(dist.name()),
};
match &dist {
Dist::Built(BuiltDist::Registry(wheel)) => {
if no_binary {
return Err(Error::NoBinary);
}
let url = match &wheel.file.url {
FileLocation::RelativeUrl(base, url) => {
pypi_types::base_url_join_relative(base, url)?
}
FileLocation::AbsoluteUrl(url) => {
Url::parse(url).map_err(|err| Error::Url(url.clone(), err))?
}
FileLocation::Path(path) => {
let url = Url::from_file_path(path).expect("path is absolute");
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Url(&url).remote_wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
// If the file is already unzipped, and the unzipped directory is fresh,
// return it.
match cache_entry.path().canonicalize() {
Ok(archive) => {
if let (Some(cache_metadata), Some(path_metadata)) = (
metadata_if_exists(&archive).map_err(Error::CacheRead)?,
metadata_if_exists(path).map_err(Error::CacheRead)?,
) {
let cache_modified = Timestamp::from_metadata(&cache_metadata);
let path_modified = Timestamp::from_metadata(&path_metadata);
if cache_modified >= path_modified {
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
archive,
filename: wheel.filename.clone(),
}));
}
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
// Otherwise, unzip the file.
return Ok(LocalWheel::Disk(DiskWheel {
dist: dist.clone(),
path: path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}));
}
};
// Create an entry for the wheel itself alongside its HTTP cache.
let wheel_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Index(&wheel.index).remote_wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
let http_entry = wheel_entry.with_file(format!("{}.http", wheel.filename.stem()));
let download = |response: reqwest::Response| {
async {
let reader = response
.bytes_stream()
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
.into_async_read();
// Download and unzip the wheel to a temporary directory.
let temp_dir =
tempfile::tempdir_in(self.cache.root()).map_err(Error::CacheWrite)?;
uv_extract::stream::unzip(reader.compat(), temp_dir.path()).await?;
// Persist the temporary directory to the directory store.
let archive = self
.cache
.persist(temp_dir.into_path(), wheel_entry.path())
.map_err(Error::CacheRead)?;
Ok(archive)
}
.instrument(info_span!("download", wheel = %wheel))
};
let req = self.client.cached_client().uncached().get(url).build()?;
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()
.get_serde(req, &http_entry, cache_control, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?;
Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
archive,
filename: wheel.filename.clone(),
}))
}
Dist::Built(BuiltDist::DirectUrl(wheel)) => {
if no_binary {
return Err(Error::NoBinary);
}
// Create an entry for the wheel itself alongside its HTTP cache.
let wheel_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Url(&wheel.url).remote_wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
let http_entry = wheel_entry.with_file(format!("{}.http", wheel.filename.stem()));
let download = |response: reqwest::Response| {
async {
let reader = response
.bytes_stream()
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
.into_async_read();
// Download and unzip the wheel to a temporary directory.
let temp_dir =
tempfile::tempdir_in(self.cache.root()).map_err(Error::CacheWrite)?;
uv_extract::stream::unzip(reader.compat(), temp_dir.path()).await?;
// Persist the temporary directory to the directory store.
let archive = self
.cache
.persist(temp_dir.into_path(), wheel_entry.path())
.map_err(Error::CacheRead)?;
Ok(archive)
}
.instrument(info_span!("download", wheel = %wheel))
};
let req = self
.client
.cached_client()
.uncached()
.get(wheel.url.raw().clone())
.build()?;
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.cache
.freshness(&http_entry, Some(wheel.name()))
.map_err(Error::CacheRead)?,
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()
.get_serde(req, &http_entry, cache_control, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?;
Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
archive,
filename: wheel.filename.clone(),
}))
}
Dist::Built(BuiltDist::Path(wheel)) => {
if no_binary {
return Err(Error::NoBinary);
}
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Url(&wheel.url).remote_wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
// If the file is already unzipped, and the unzipped directory is fresh,
// return it.
match cache_entry.path().canonicalize() {
Ok(archive) => {
if let (Some(cache_metadata), Some(path_metadata)) = (
metadata_if_exists(&archive).map_err(Error::CacheRead)?,
metadata_if_exists(&wheel.path).map_err(Error::CacheRead)?,
) {
let cache_modified = Timestamp::from_metadata(&cache_metadata);
let path_modified = Timestamp::from_metadata(&path_metadata);
if cache_modified >= path_modified {
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
archive,
filename: wheel.filename.clone(),
}));
}
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
Ok(LocalWheel::Disk(DiskWheel {
dist: dist.clone(),
path: wheel.path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}))
}
Dist::Source(source_dist) => {
let lock = self.locks.acquire(&dist).await;
let _guard = lock.lock().await;
let built_wheel = self.builder.download_and_build(source_dist).boxed().await?;
// If the wheel was unzipped previously, respect it. Source distributions are
// cached under a unique build ID, so unzipped directories are never stale.
match built_wheel.target.canonicalize() {
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
archive,
filename: built_wheel.filename,
})),
Err(err) if err.kind() == io::ErrorKind::NotFound => {
Ok(LocalWheel::Built(BuiltWheel {
dist: dist.clone(),
path: built_wheel.path,
target: built_wheel.target,
filename: built_wheel.filename,
}))
}
Err(err) => return Err(Error::CacheRead(err)),
}
}
}
}
/// Either fetch the only wheel metadata (directly from the index or with range requests) or
/// fetch and build the source distribution.
///
/// Returns the [`Metadata21`], along with a "precise" URL for the source distribution, if
/// possible. For example, given a Git dependency with a reference to a branch or tag, return a
/// URL with a precise reference to the current commit of that branch or tag.
#[instrument(skip_all, fields(%dist))]
pub async fn get_or_build_wheel_metadata(
&self,
dist: &Dist,
) -> Result<(Metadata21, Option<Url>), Error> {
match dist {
Dist::Built(built_dist) => {
Ok((self.client.wheel_metadata(built_dist).boxed().await?, None))
}
Dist::Source(source_dist) => {
let no_build = match self.build_context.no_build() {
NoBuild::All => true,
NoBuild::None => false,
NoBuild::Packages(packages) => packages.contains(source_dist.name()),
};
// Optimization: Skip source dist download when we must not build them anyway.
if no_build {
return Err(Error::NoBuild);
}
let lock = self.locks.acquire(dist).await;
let _guard = lock.lock().await;
// Insert the `precise` URL, if it exists.
let precise = self.precise(source_dist).await?;
let source_dist = match precise.as_ref() {
Some(url) => Cow::Owned(source_dist.clone().with_url(url.clone())),
None => Cow::Borrowed(source_dist),
};
let metadata = self
.builder
.download_and_build_metadata(&source_dist)
.boxed()
.await?;
Ok((metadata, precise))
}
}
}
/// Build a directory into an editable wheel.
pub async fn build_wheel_editable(
&self,
editable: &LocalEditable,
editable_wheel_dir: &Path,
) -> Result<(LocalWheel, Metadata21), Error> {
let (dist, disk_filename, filename, metadata) = self
.builder
.build_editable(editable, editable_wheel_dir)
.await?;
let built_wheel = BuiltWheel {
dist,
filename,
path: editable_wheel_dir.join(disk_filename),
target: editable_wheel_dir.join(cache_key::digest(&editable.path)),
};
Ok((LocalWheel::Built(built_wheel), metadata))
}
/// Given a remote source distribution, return a precise variant, if possible.
///
/// For example, given a Git dependency with a reference to a branch or tag, return a URL
/// with a precise reference to the current commit of that branch or tag.
///
/// This method takes into account various normalizations that are independent from the Git
/// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+`
/// prefix kinds.
async fn precise(&self, dist: &SourceDist) -> Result<Option<Url>, Error> {
let SourceDist::Git(source_dist) = dist else {
return Ok(None);
};
let git_dir = self.build_context.cache().bucket(CacheBucket::Git);
let DirectGitUrl { url, subdirectory } =
DirectGitUrl::try_from(source_dist.url.raw()).map_err(Error::Git)?;
// If the commit already contains a complete SHA, short-circuit.
if url.precise().is_some() {
return Ok(None);
}
// Fetch the precise SHA of the Git reference (which could be a branch, a tag, a partial
// commit, etc.).
let source = if let Some(reporter) = self.reporter.clone() {
GitSource::new(url, git_dir).with_reporter(Facade::from(reporter))
} else {
GitSource::new(url, git_dir)
};
let precise = tokio::task::spawn_blocking(move || source.fetch())
.await?
.map_err(Error::Git)?;
let url = precise.into_git();
// Re-encode as a URL.
Ok(Some(Url::from(DirectGitUrl { url, subdirectory })))
}
pub fn index_locations(&self) -> &IndexLocations {
self.build_context.index_locations()
}
}

127
crates/uv-distribution/src/download.rs Normal file
View file

@ -0,0 +1,127 @@
use std::path::{Path, PathBuf};
use distribution_filename::WheelFilename;
use distribution_types::{CachedDist, Dist};
/// A wheel that's been unzipped while downloading
#[derive(Debug, Clone)]
pub struct UnzippedWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The canonicalized path in the cache directory to which the wheel was downloaded.
/// Typically, a directory within the archive bucket.
pub(crate) archive: PathBuf,
}
/// A downloaded wheel that's stored on-disk.
#[derive(Debug, Clone)]
pub struct DiskWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the downloaded wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A wheel built from a source distribution that's stored on-disk.
#[derive(Debug, Clone)]
pub struct BuiltWheel {
/// The remote source distribution from which this wheel was built.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A downloaded or built wheel.
#[derive(Debug, Clone)]
pub enum LocalWheel {
Unzipped(UnzippedWheel),
Disk(DiskWheel),
Built(BuiltWheel),
}
impl LocalWheel {
/// Return the path to the downloaded wheel's entry in the cache.
pub fn target(&self) -> &Path {
match self {
LocalWheel::Unzipped(wheel) => &wheel.archive,
LocalWheel::Disk(wheel) => &wheel.target,
LocalWheel::Built(wheel) => &wheel.target,
}
}
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
match self {
LocalWheel::Unzipped(wheel) => wheel.remote(),
LocalWheel::Disk(wheel) => wheel.remote(),
LocalWheel::Built(wheel) => wheel.remote(),
}
}
/// Return the [`WheelFilename`] of this wheel.
pub fn filename(&self) -> &WheelFilename {
match self {
LocalWheel::Unzipped(wheel) => &wheel.filename,
LocalWheel::Disk(wheel) => &wheel.filename,
LocalWheel::Built(wheel) => &wheel.filename,
}
}
/// Convert a [`LocalWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self, archive: PathBuf) -> CachedDist {
match self {
LocalWheel::Unzipped(wheel) => {
CachedDist::from_remote(wheel.dist, wheel.filename, archive)
}
LocalWheel::Disk(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
LocalWheel::Built(wheel) => {
CachedDist::from_remote(wheel.dist, wheel.filename, archive)
}
}
}
}
impl UnzippedWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
/// Convert an [`UnzippedWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self) -> CachedDist {
CachedDist::from_remote(self.dist, self.filename, self.archive)
}
}
impl DiskWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
}
impl BuiltWheel {
/// Return the [`Dist`] from which this source distribution that this wheel was built from was
/// downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
}
impl std::fmt::Display for LocalWheel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.remote())
}
}

62
crates/uv-distribution/src/error.rs Normal file
View file

@ -0,0 +1,62 @@
use tokio::task::JoinError;
use zip::result::ZipError;
use distribution_filename::WheelFilenameError;
use uv_normalize::PackageName;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("Building source distributions is disabled")]
NoBuild,
#[error("Using pre-built wheels is disabled")]
NoBinary,
// Network error
#[error("Failed to parse URL: `{0}`")]
Url(String, #[source] url::ParseError),
#[error(transparent)]
JoinRelativeUrl(#[from] pypi_types::JoinRelativeError),
#[error("Git operation failed")]
Git(#[source] anyhow::Error),
#[error(transparent)]
Request(#[from] reqwest::Error),
#[error(transparent)]
Client(#[from] uv_client::Error),
// Cache writing error
#[error("Failed to read from the distribution cache")]
CacheRead(#[source] std::io::Error),
#[error("Failed to write to the distribution cache")]
CacheWrite(#[source] std::io::Error),
#[error("Failed to deserialize cache entry")]
CacheDecode(#[from] rmp_serde::decode::Error),
#[error("Failed to serialize cache entry")]
CacheEncode(#[from] rmp_serde::encode::Error),
// Build error
#[error("Failed to build: {0}")]
Build(String, #[source] anyhow::Error),
#[error("Failed to build editable: {0}")]
BuildEditable(String, #[source] anyhow::Error),
#[error("Built wheel has an invalid filename")]
WheelFilename(#[from] WheelFilenameError),
#[error("Package metadata name `{metadata}` does not match given name `{given}`")]
NameMismatch {
given: PackageName,
metadata: PackageName,
},
#[error("Failed to parse metadata from built wheel")]
Metadata(#[from] pypi_types::Error),
#[error("Failed to read `dist-info` metadata from built wheel")]
DistInfo(#[from] install_wheel_rs::Error),
#[error("Failed to read zip archive from built wheel")]
Zip(#[from] ZipError),
#[error("Source distribution directory contains neither readable pyproject.toml nor setup.py")]
DirWithoutEntrypoint,
#[error("Failed to extract source distribution: {0}")]
Extract(#[from] uv_extract::Error),
/// Should not occur; only seen when another task panicked.
#[error("The task executor is broken, did some other task panic?")]
Join(#[from] JoinError),
}

129
crates/uv-distribution/src/index/built_wheel_index.rs Normal file
View file

@ -0,0 +1,129 @@
use distribution_types::{git_reference, DirectUrlSourceDist, GitSourceDist, Name, PathSourceDist};
use platform_tags::Tags;
use uv_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, WheelCache};
use uv_fs::symlinks;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, read_timestamp_manifest, MANIFEST};
use crate::Error;
/// A local index of built distributions for a specific source distribution.
pub struct BuiltWheelIndex;
impl BuiltWheelIndex {
/// Return the most compatible [`CachedWheel`] for a given source distribution at a direct URL.
///
/// This method does not perform any freshness checks and assumes that the source distribution
/// is already up-to-date.
pub fn url(
source_dist: &DirectUrlSourceDist,
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Url(source_dist.url.raw()).remote_wheel_dir(source_dist.name().as_ref()),
);
// Read the manifest from the cache. There's no need to enforce freshness, since we
// enforce freshness on the entries.
let manifest_entry = cache_shard.entry(MANIFEST);
let Some(manifest) = read_http_manifest(&manifest_entry)? else {
return Ok(None);
};
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a local path.
pub fn path(
source_dist: &PathSourceDist,
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
let cache_shard = cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Path(&source_dist.url).remote_wheel_dir(source_dist.name().as_ref()),
);
// Determine the last-modified time of the source distribution.
let Some(modified) = ArchiveTimestamp::from_path(&source_dist.path).expect("archived")
else {
return Err(Error::DirWithoutEntrypoint);
};
// Read the manifest from the cache. There's no need to enforce freshness, since we
// enforce freshness on the entries.
let manifest_entry = cache_shard.entry(MANIFEST);
let Some(manifest) = read_timestamp_manifest(&manifest_entry, modified)? else {
return Ok(None);
};
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL.
pub fn git(source_dist: &GitSourceDist, cache: &Cache, tags: &Tags) -> Option<CachedWheel> {
let Ok(Some(git_sha)) = git_reference(&source_dist.url) else {
return None;
};
let cache_shard = cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Git(&source_dist.url, &git_sha.to_short_string())
.remote_wheel_dir(source_dist.name().as_ref()),
);
Self::find(&cache_shard, tags)
}
/// Find the "best" distribution in the index for a given source distribution.
///
/// This lookup prefers newer versions over older versions, and aims to maximize compatibility
/// with the target platform.
///
/// The `shard` should point to a directory containing the built distributions for a specific
/// source distribution. For example, given the built wheel cache structure:
/// ```text
/// built-wheels-v0/
/// └── pypi
/// └── django-allauth-0.51.0.tar.gz
/// ├── django_allauth-0.51.0-py3-none-any.whl
/// └── metadata.json
/// ```
///
/// The `shard` should be `built-wheels-v0/pypi/django-allauth-0.51.0.tar.gz`.
fn find(shard: &CacheShard, tags: &Tags) -> Option<CachedWheel> {
let mut candidate: Option<CachedWheel> = None;
// Unzipped wheels are stored as symlinks into the archive directory.
for subdir in symlinks(shard) {
match CachedWheel::from_path(&subdir) {
None => {}
Some(dist_info) => {
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
// Only consider wheels that are compatible with our tags.
if !compatibility.is_compatible() {
continue;
}
if let Some(existing) = candidate.as_ref() {
// Override if the wheel is newer, or "more" compatible.
if dist_info.filename.version > existing.filename.version
|| compatibility > existing.filename.compatibility(tags)
{
candidate = Some(dist_info);
}
} else {
candidate = Some(dist_info);
}
}
}
}
candidate
}
}

43
crates/uv-distribution/src/index/cached_wheel.rs Normal file
View file

@ -0,0 +1,43 @@
use std::path::Path;
use distribution_filename::WheelFilename;
use distribution_types::{CachedDirectUrlDist, CachedRegistryDist};
use pep508_rs::VerbatimUrl;
use uv_cache::CacheEntry;
#[derive(Debug, Clone)]
pub struct CachedWheel {
/// The filename of the wheel.
pub filename: WheelFilename,
/// The [`CacheEntry`] for the wheel.
pub entry: CacheEntry,
}
impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_path(path: &Path) -> Option<Self> {
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive);
Some(Self { filename, entry })
}
/// Convert a [`CachedWheel`] into a [`CachedRegistryDist`].
pub fn into_registry_dist(self) -> CachedRegistryDist {
CachedRegistryDist {
filename: self.filename,
path: self.entry.into_path_buf(),
}
}
/// Convert a [`CachedWheel`] into a [`CachedDirectUrlDist`].
pub fn into_url_dist(self, url: VerbatimUrl) -> CachedDirectUrlDist {
CachedDirectUrlDist {
filename: self.filename,
url,
path: self.entry.into_path_buf(),
editable: false,
}
}
}

6
crates/uv-distribution/src/index/mod.rs Normal file
View file

@ -0,0 +1,6 @@
pub use built_wheel_index::BuiltWheelIndex;
pub use registry_wheel_index::RegistryWheelIndex;
mod built_wheel_index;
mod cached_wheel;
mod registry_wheel_index;

149
crates/uv-distribution/src/index/registry_wheel_index.rs Normal file
View file

@ -0,0 +1,149 @@
use std::collections::hash_map::Entry;
use std::collections::BTreeMap;
use std::path::Path;
use rustc_hash::FxHashMap;
use distribution_types::{CachedRegistryDist, FlatIndexLocation, IndexLocations, IndexUrl};
use pep440_rs::Version;
use platform_tags::Tags;
use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_fs::{directories, symlinks};
use uv_normalize::PackageName;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, MANIFEST};
/// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)]
pub struct RegistryWheelIndex<'a> {
cache: &'a Cache,
tags: &'a Tags,
index_locations: &'a IndexLocations,
index: FxHashMap<&'a PackageName, BTreeMap<Version, CachedRegistryDist>>,
}
impl<'a> RegistryWheelIndex<'a> {
/// Initialize an index of cached distributions from a directory.
pub fn new(cache: &'a Cache, tags: &'a Tags, index_locations: &'a IndexLocations) -> Self {
Self {
cache,
tags,
index_locations,
index: FxHashMap::default(),
}
}
/// Return an iterator over available wheels for a given package.
///
/// If the package is not yet indexed, this will index the package by reading from the cache.
pub fn get(
&mut self,
name: &'a PackageName,
) -> impl Iterator<Item = (&Version, &CachedRegistryDist)> {
self.get_impl(name).iter().rev()
}
/// Get the best wheel for the given package name and version.
///
/// If the package is not yet indexed, this will index the package by reading from the cache.
pub fn get_version(
&mut self,
name: &'a PackageName,
version: &Version,
) -> Option<&CachedRegistryDist> {
self.get_impl(name).get(version)
}
/// Get an entry in the index.
fn get_impl(&mut self, name: &'a PackageName) -> &BTreeMap<Version, CachedRegistryDist> {
let versions = match self.index.entry(name) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => entry.insert(Self::index(
name,
self.cache,
self.tags,
self.index_locations,
)),
};
versions
}
/// Add a package to the index by reading from the cache.
fn index(
package: &PackageName,
cache: &Cache,
tags: &Tags,
index_locations: &IndexLocations,
) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new();
// Collect into owned `IndexUrl`
let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index()
.filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(_) => None,
FlatIndexLocation::Url(url) => Some(IndexUrl::Url(url.clone())),
})
.collect();
for index_url in index_locations.indexes().chain(flat_index_urls.iter()) {
// Index all the wheels that were downloaded directly from the registry.
let wheel_dir = cache.shard(
CacheBucket::Wheels,
WheelCache::Index(index_url).remote_wheel_dir(package.to_string()),
);
Self::add_directory(&wheel_dir, tags, &mut versions);
// Index all the built wheels, created by downloading and building source distributions
// from the registry.
let cache_shard = cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Index(index_url).built_wheel_dir(package.to_string()),
);
// For registry wheels, the cache structure is: `<index>/<package-name>/<version>/`.
for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard);
let manifest_entry = cache_shard.entry(MANIFEST);
if let Ok(Some(manifest)) = read_http_manifest(&manifest_entry) {
Self::add_directory(cache_shard.join(manifest.id()), tags, &mut versions);
};
}
}
versions
}
/// Add the wheels in a given directory to the index.
///
/// Each subdirectory in the given path is expected to be that of an unzipped wheel.
fn add_directory(
path: impl AsRef<Path>,
tags: &Tags,
versions: &mut BTreeMap<Version, CachedRegistryDist>,
) {
// Unzipped wheels are stored as symlinks into the archive directory.
for wheel_dir in symlinks(path.as_ref()) {
match CachedWheel::from_path(&wheel_dir) {
None => {}
Some(dist_info) => {
let dist_info = dist_info.into_registry_dist();
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
if let Some(existing) = versions.get_mut(&dist_info.filename.version) {
// Override if we have better compatibility
if compatibility > existing.filename.compatibility(tags) {
*existing = dist_info;
}
} else if compatibility.is_compatible() {
versions.insert(dist_info.filename.version.clone(), dist_info);
}
}
}
}
}
}

16
crates/uv-distribution/src/lib.rs Normal file
View file

@ -0,0 +1,16 @@
pub use distribution_database::DistributionDatabase;
pub use download::{BuiltWheel, DiskWheel, LocalWheel};
pub use error::Error;
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
pub use reporter::Reporter;
pub use source::SourceDistCachedBuilder;
pub use unzip::Unzip;
mod distribution_database;
mod download;
mod error;
mod index;
mod locks;
mod reporter;
mod source;
mod unzip;

20
crates/uv-distribution/src/locks.rs Normal file
View file

@ -0,0 +1,20 @@
use std::sync::Arc;
use rustc_hash::FxHashMap;
use tokio::sync::Mutex;
use distribution_types::{Identifier, ResourceId};
/// A set of locks used to prevent concurrent access to the same resource.
#[derive(Debug, Default)]
pub(crate) struct Locks(Mutex<FxHashMap<ResourceId, Arc<Mutex<()>>>>);
impl Locks {
/// Acquire a lock on the given resource.
pub(crate) async fn acquire(&self, dist: &impl Identifier) -> Arc<Mutex<()>> {
let mut map = self.0.lock().await;
map.entry(dist.resource_id())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone()
}
}

40
crates/uv-distribution/src/reporter.rs Normal file
View file

@ -0,0 +1,40 @@
use std::sync::Arc;
use url::Url;
use distribution_types::SourceDist;
pub trait Reporter: Send + Sync {
/// Callback to invoke when a source distribution build is kicked off.
fn on_build_start(&self, dist: &SourceDist) -> usize;
/// Callback to invoke when a source distribution build is complete.
fn on_build_complete(&self, dist: &SourceDist, id: usize);
/// Callback to invoke when a repository checkout begins.
fn on_checkout_start(&self, url: &Url, rev: &str) -> usize;
/// Callback to invoke when a repository checkout completes.
fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize);
}
/// A facade for converting from [`Reporter`] to [`uv_git::Reporter`].
pub(crate) struct Facade {
reporter: Arc<dyn Reporter>,
}
impl From<Arc<dyn Reporter>> for Facade {
fn from(reporter: Arc<dyn Reporter>) -> Self {
Self { reporter }
}
}
impl uv_git::Reporter for Facade {
fn on_checkout_start(&self, url: &Url, rev: &str) -> usize {
self.reporter.on_checkout_start(url, rev)
}
fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize) {
self.reporter.on_checkout_complete(url, rev, index);
}
}

44
crates/uv-distribution/src/source/built_wheel_metadata.rs Normal file
View file

@ -0,0 +1,44 @@
use std::path::PathBuf;
use std::str::FromStr;
use distribution_filename::WheelFilename;
use platform_tags::Tags;
use uv_cache::CacheShard;
use uv_fs::files;
/// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)]
pub struct BuiltWheelMetadata {
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
pub(crate) target: PathBuf,
/// The parsed filename.
pub(crate) filename: WheelFilename,
}
impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest.
pub(crate) fn find_in_cache(tags: &Tags, cache_shard: &CacheShard) -> Option<Self> {
for directory in files(cache_shard) {
if let Some(metadata) = Self::from_path(directory, cache_shard) {
// Validate that the wheel is compatible with the target platform.
if metadata.filename.is_compatible(tags) {
return Some(metadata);
}
}
}
None
}
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any.whl`).
fn from_path(path: PathBuf, cache_shard: &CacheShard) -> Option<Self> {
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_str(filename).ok()?;
Some(Self {
target: cache_shard.join(filename.stem()),
path,
filename,
})
}
}

17
crates/uv-distribution/src/source/manifest.rs Normal file
View file

@ -0,0 +1,17 @@
use serde::{Deserialize, Serialize};
/// The [`Manifest`] is a thin wrapper around a unique identifier for the source distribution.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct Manifest(String);
impl Manifest {
/// Initialize a new [`Manifest`] with a random UUID.
pub(crate) fn new() -> Self {
Self(nanoid::nanoid!())
}
/// Return the unique ID of the manifest.
pub(crate) fn id(&self) -> &str {
&self.0
}
}

1063
crates/uv-distribution/src/source/mod.rs Normal file
View file

File diff suppressed because it is too large Load diff

33
crates/uv-distribution/src/unzip.rs Normal file
View file

@ -0,0 +1,33 @@
use std::path::Path;
use uv_extract::Error;
use crate::download::BuiltWheel;
use crate::{DiskWheel, LocalWheel};
pub trait Unzip {
/// Unzip a wheel into the target directory.
fn unzip(&self, target: &Path) -> Result<(), Error>;
}
impl Unzip for DiskWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for BuiltWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for LocalWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
match self {
LocalWheel::Unzipped(_) => Ok(()),
LocalWheel::Disk(wheel) => wheel.unzip(target),
LocalWheel::Built(wheel) => wheel.unzip(target),
}
}
}