Remove special-casing for editable requirements (#3869)

## Summary

There are a few behavior changes in here:

- We now enforce `--require-hashes` for editables, like pip. So if you
use `--require-hashes` with an editable requirement, we'll reject it. I
could change this if it seems off.
- We now treat source tree requirements, editable or not (e.g., both `-e
./black` and `./black`) as if `--refresh` is always enabled. This
doesn't mean that we _always_ rebuild them; but if you pass
`--reinstall`, then yes, we always rebuild them. I think this is an
improvement and is close to how editables work today.

Closes #3844.

Closes #2695.
This commit is contained in:
Charlie Marsh 2024-05-28 11:49:34 -04:00 committed by GitHub
parent 063a0a4384
commit 1fc6a59707
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
64 changed files with 583 additions and 1813 deletions

View file

@ -11,16 +11,16 @@ use tempfile::TempDir;
use tokio::io::{AsyncRead, AsyncSeekExt, ReadBuf};
use tokio::sync::Semaphore;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info_span, instrument, warn, Instrument};
use tracing::{debug, info_span, instrument, warn, Instrument};
use url::Url;
use distribution_filename::WheelFilename;
use distribution_types::{
BuildableSource, BuiltDist, Dist, FileLocation, HashPolicy, Hashed, IndexLocations,
LocalEditable, Name, SourceDist,
BuildableSource, BuiltDist, Dist, FileLocation, HashPolicy, Hashed, IndexLocations, Name,
SourceDist,
};
use platform_tags::Tags;
use pypi_types::{HashDigest, Metadata23};
use pypi_types::HashDigest;
use uv_cache::{ArchiveId, ArchiveTimestamp, CacheBucket, CacheEntry, Timestamp, WheelCache};
use uv_client::{
CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
@ -133,32 +133,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
}
}
/// Build a directory into an editable wheel.
pub async fn build_wheel_editable(
&self,
editable: &LocalEditable,
editable_wheel_dir: &Path,
) -> Result<(LocalWheel, Metadata23), Error> {
// Build the wheel.
let (dist, disk_filename, filename, metadata) = self
.builder
.build_editable(editable, editable_wheel_dir)
.await?;
// Unzip into the editable wheel directory.
let path = editable_wheel_dir.join(&disk_filename);
let target = editable_wheel_dir.join(cache_key::digest(&editable.path));
let id = self.unzip_wheel(&path, &target).await?;
let wheel = LocalWheel {
dist,
filename,
archive: self.build_context.cache().archive(&id),
hashes: vec![],
};
Ok((wheel, metadata))
}
/// Fetch a wheel from the cache or download it from the index.
///
/// While hashes will be generated in all cases, hash-checking is _not_ enforced and should
@ -432,7 +406,11 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
// Optimization: Skip source dist download when we must not build them anyway.
if no_build {
return Err(Error::NoBuild);
if source.is_editable() {
debug!("Allowing build for editable source distribution: {source}");
} else {
return Err(Error::NoBuild);
}
}
let lock = self.locks.acquire(source).await;
@ -443,6 +421,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
.download_and_build_metadata(source, hashes, &self.client)
.boxed_local()
.await?;
Ok(metadata)
}

View file

@ -92,7 +92,11 @@ impl<'a> BuiltWheelIndex<'a> {
) -> Result<Option<CachedWheel>, Error> {
let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Path(&source_dist.url).root(),
if source_dist.editable {
WheelCache::Editable(&source_dist.url).root()
} else {
WheelCache::Path(&source_dist.url).root()
},
);
// Read the revision from the cache.

View file

@ -16,8 +16,8 @@ use zip::ZipArchive;
use distribution_filename::WheelFilename;
use distribution_types::{
BuildableSource, DirectorySourceDist, DirectorySourceUrl, Dist, FileLocation, GitSourceUrl,
HashPolicy, Hashed, LocalEditable, PathSourceUrl, RemoteSource, SourceDist, SourceUrl,
BuildableSource, DirectorySourceUrl, FileLocation, GitSourceUrl, HashPolicy, Hashed,
PathSourceUrl, RemoteSource, SourceDist, SourceUrl,
};
use install_wheel_rs::metadata::read_archive_metadata;
use platform_tags::Tags;
@ -369,7 +369,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.boxed_local()
.await?
}
BuildableSource::Url(SourceUrl::Path(resource)) => {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
@ -825,7 +824,8 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
Ok(revision)
}
/// Build a source distribution from a local source tree (i.e., directory).
/// Build a source distribution from a local source tree (i.e., directory), either editable or
/// non-editable.
async fn source_tree(
&self,
source: &BuildableSource<'_>,
@ -840,15 +840,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
if resource.editable {
WheelCache::Editable(resource.url).root()
} else {
WheelCache::Path(resource.url).root()
},
);
let _lock = lock_shard(&cache_shard).await?;
// Fetch the revision for the source distribution.
let revision = self
.source_tree_revision(source, resource, &cache_shard)
.await?;
let revision = self.source_tree_revision(resource, &cache_shard).await?;
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
@ -889,7 +891,8 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
})
}
/// Build the source distribution's metadata from a local source tree (i.e., a directory).
/// Build the source distribution's metadata from a local source tree (i.e., a directory),
/// either editable or non-editable.
///
/// If the build backend supports `prepare_metadata_for_build_wheel`, this method will avoid
/// building the wheel.
@ -906,15 +909,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let cache_shard = self.build_context.cache().shard(
CacheBucket::BuiltWheels,
WheelCache::Path(resource.url).root(),
if resource.editable {
WheelCache::Editable(resource.url).root()
} else {
WheelCache::Path(resource.url).root()
},
);
let _lock = lock_shard(&cache_shard).await?;
// Fetch the revision for the source distribution.
let revision = self
.source_tree_revision(source, resource, &cache_shard)
.await?;
let revision = self.source_tree_revision(resource, &cache_shard).await?;
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
@ -971,7 +976,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
/// Return the [`Revision`] for a local source tree, refreshing it if necessary.
async fn source_tree_revision(
&self,
source: &BuildableSource<'_>,
resource: &DirectorySourceUrl<'_>,
cache_shard: &CacheShard,
) -> Result<Revision, Error> {
@ -982,16 +986,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
return Err(Error::DirWithoutEntrypoint(resource.path.to_path_buf()));
};
// Read the existing metadata from the cache.
// Read the existing metadata from the cache. We treat source trees as if `--refresh` is
// always set, since they're mutable.
let entry = cache_shard.entry(LOCAL_REVISION);
let freshness = self
let is_fresh = self
.build_context
.cache()
.freshness(&entry, source.name())
.is_fresh(&entry)
.map_err(Error::CacheRead)?;
// If the revision is fresh, return it.
if freshness.is_fresh() {
if is_fresh {
if let Some(pointer) = LocalRevisionPointer::read_from(&entry)? {
if pointer.timestamp == modified.timestamp() {
return Ok(pointer.into_revision());
@ -1299,8 +1304,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source.name().is_some_and(|name| packages.contains(name))
}
};
if no_build {
return Err(Error::NoBuild);
if source.is_editable() {
debug!("Allowing build for editable source distribution: {source}");
} else {
return Err(Error::NoBuild);
}
}
// Build the wheel.
@ -1314,7 +1324,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
subdirectory,
&source.to_string(),
source.as_dist(),
BuildKind::Wheel,
if source.is_editable() {
BuildKind::Editable
} else {
BuildKind::Wheel
},
)
.await
.map_err(|err| Error::Build(source.to_string(), err))?
@ -1383,7 +1397,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
subdirectory,
&source.to_string(),
source.as_dist(),
BuildKind::Wheel,
if source.is_editable() {
BuildKind::Editable
} else {
BuildKind::Wheel
},
)
.await
.map_err(|err| Error::Build(source.to_string(), err))?;
@ -1410,49 +1428,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
Ok(Some(metadata))
}
/// Build a single directory into an editable wheel
pub async fn build_editable(
&self,
editable: &LocalEditable,
editable_wheel_dir: &Path,
) -> Result<(Dist, String, WheelFilename, Metadata23), Error> {
debug!("Building (editable) {editable}");
// Verify that the editable exists.
if !editable.path.exists() {
return Err(Error::NotFound(editable.path.clone()));
}
// Build the wheel.
let disk_filename = self
.build_context
.setup_build(
&editable.path,
None,
&editable.to_string(),
None,
BuildKind::Editable,
)
.await
.map_err(|err| Error::BuildEditable(editable.to_string(), err))?
.wheel(editable_wheel_dir)
.await
.map_err(|err| Error::BuildEditable(editable.to_string(), err))?;
let filename = WheelFilename::from_str(&disk_filename)?;
// We finally have the name of the package and can construct the dist.
let dist = Dist::Source(SourceDist::Directory(DirectorySourceDist {
name: filename.name.clone(),
url: editable.url().clone(),
path: editable.path.clone(),
editable: true,
}));
let metadata = read_wheel_metadata(&filename, editable_wheel_dir.join(&disk_filename))?;
debug!("Finished building (editable): {dist}");
Ok((dist, disk_filename, filename, metadata))
}
/// Returns a GET [`reqwest::Request`] for the given URL.
fn request(url: Url, client: &RegistryClient) -> Result<reqwest::Request, reqwest::Error> {
client