mirror of
https://github.com/astral-sh/uv.git
synced 2025-10-28 10:50:29 +00:00
Cache metadata for source tree dependencies (#5423)
## Summary This PR re-introduces caching for source trees. In short, we treat the metadata as cached unless the `pyproject.toml`, `setup.py`, or `setup.cfg` file changes. This is a heuristic and not a good one, especially for extension modules, but without it, we have to rebuild every project every time (unless you have static metadata, like a `pyproject.toml` that we can read directly). Now that we support persistent configuration, users should add: ```toml [tool.uv] reinstall = ["foo"] ``` If they want a package to always be refreshed (ignore cache) and reinstalled (ignore environment). Closes https://github.com/astral-sh/uv/issues/5420.
This commit is contained in:
Charlie Marsh
GitHub
parent
375a4152fa
commit
4d9098a1d7
5 changed files with 72 additions and 28 deletions
|
|
@ -22,8 +22,7 @@ use install_wheel_rs::metadata::read_archive_metadata;
|
|||
use platform_tags::Tags;
|
||||
use pypi_types::{HashDigest, Metadata23, ParsedArchiveUrl};
|
||||
use uv_cache::{
|
||||
ArchiveTimestamp, CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Freshness, Timestamp,
|
||||
WheelCache,
|
||||
ArchiveTimestamp, CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Timestamp, WheelCache,
|
||||
};
|
||||
use uv_client::{
|
||||
CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
|
||||
|
|
@ -903,7 +902,9 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
|
|||
let _lock = lock_shard(&cache_shard).await?;
|
||||
|
||||
// Fetch the revision for the source distribution.
|
||||
let revision = self.source_tree_revision(resource, &cache_shard).await?;
|
||||
let revision = self
|
||||
.source_tree_revision(source, resource, &cache_shard)
|
||||
.await?;
|
||||
|
||||
// Scope all operations to the revision. Within the revision, there's no need to check for
|
||||
// freshness, since entries have to be fresher than the revision itself.
|
||||
|
|
@ -972,7 +973,9 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
|
|||
let _lock = lock_shard(&cache_shard).await?;
|
||||
|
||||
// Fetch the revision for the source distribution.
|
||||
let revision = self.source_tree_revision(resource, &cache_shard).await?;
|
||||
let revision = self
|
||||
.source_tree_revision(source, resource, &cache_shard)
|
||||
.await?;
|
||||
|
||||
// Scope all operations to the revision. Within the revision, there's no need to check for
|
||||
// freshness, since entries have to be fresher than the revision itself.
|
||||
|
|
@ -1053,6 +1056,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
|
|||
/// Return the [`Revision`] for a local source tree, refreshing it if necessary.
|
||||
async fn source_tree_revision(
|
||||
&self,
|
||||
source: &BuildableSource<'_>,
|
||||
resource: &DirectorySourceUrl<'_>,
|
||||
cache_shard: &CacheShard,
|
||||
) -> Result<Revision, Error> {
|
||||
|
|
@ -1070,17 +1074,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
|
|||
));
|
||||
};
|
||||
|
||||
// Read the existing metadata from the cache. We treat source trees as if `--refresh` is
|
||||
// always set, since they're mutable.
|
||||
// Read the existing metadata from the cache.
|
||||
let entry = cache_shard.entry(LOCAL_REVISION);
|
||||
let is_fresh = self
|
||||
.build_context
|
||||
.cache()
|
||||
.is_fresh(&entry)
|
||||
.map_err(Error::CacheRead)?;
|
||||
|
||||
// If the revision is fresh, return it.
|
||||
if is_fresh {
|
||||
if self
|
||||
.build_context
|
||||
.cache()
|
||||
.freshness(&entry, source.name())
|
||||
.map_err(Error::CacheRead)?
|
||||
.is_fresh()
|
||||
{
|
||||
if let Some(pointer) = LocalRevisionPointer::read_from(&entry)? {
|
||||
if pointer.timestamp == modified.timestamp() {
|
||||
return Ok(pointer.into_revision());
|
||||
|
|
@ -1243,7 +1247,8 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
|
|||
.build_context
|
||||
.cache()
|
||||
.freshness(&metadata_entry, source.name())
|
||||
.is_ok_and(Freshness::is_fresh)
|
||||
.map_err(Error::CacheRead)?
|
||||
.is_fresh()
|
||||
{
|
||||
if let Some(metadata) = read_cached_metadata(&metadata_entry).await? {
|
||||
debug!("Using cached metadata for: {source}");
|
||||
|
|
|
|||
|
|
@ -10091,7 +10091,7 @@ requires-python = ">3.8"
|
|||
# via -r requirements.in
|
||||
idna==3.6
|
||||
# via anyio
|
||||
lib @ file://[TEMP_DIR]/lib/
|
||||
lib @ file://[TEMP_DIR]/lib
|
||||
# via example
|
||||
|
||||
----- stderr -----
|
||||
|
|
|
|||
|
|
@ -4489,7 +4489,6 @@ fn already_installed_dependent_editable() {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 1 package in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- first-local==0.1.0 (from file://[WORKSPACE]/scripts/packages/dependent_locals/first_local)
|
||||
|
|
@ -4588,7 +4587,6 @@ fn already_installed_local_path_dependent() {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 2 packages in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- first-local==0.1.0 (from file://[WORKSPACE]/scripts/packages/dependent_locals/first_local)
|
||||
|
|
@ -4673,6 +4671,7 @@ fn already_installed_local_version_of_remote_package() {
|
|||
);
|
||||
|
||||
// Request install with a different version
|
||||
//
|
||||
// We should attempt to pull from the index since the installed version does not match
|
||||
// but we disable it here to preserve this dependency for future tests
|
||||
uv_snapshot!(context.filters(), context.pip_install()
|
||||
|
|
@ -4705,8 +4704,8 @@ fn already_installed_local_version_of_remote_package() {
|
|||
"###
|
||||
);
|
||||
|
||||
// Request reinstall with the full path, this should reinstall from the path
|
||||
// and not pull from the index
|
||||
// Request reinstall with the full path, this should reinstall from the path and not pull from
|
||||
// the index (or rebuild).
|
||||
uv_snapshot!(context.filters(), context.pip_install()
|
||||
.arg(root_path.join("anyio_local"))
|
||||
.arg("--reinstall")
|
||||
|
|
@ -4717,7 +4716,6 @@ fn already_installed_local_version_of_remote_package() {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 1 package in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- anyio==4.3.0+foo (from file://[WORKSPACE]/scripts/packages/anyio_local)
|
||||
|
|
@ -4755,7 +4753,6 @@ fn already_installed_local_version_of_remote_package() {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 1 package in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- anyio==4.3.0
|
||||
|
|
|
|||
|
|
@ -2198,7 +2198,8 @@ fn refresh_package() -> Result<()> {
|
|||
fn sync_editable() -> Result<()> {
|
||||
let context = TestContext::new("3.12");
|
||||
let poetry_editable = context.temp_dir.child("poetry_editable");
|
||||
// Copy into the temporary directory so we can mutate it
|
||||
|
||||
// Copy into the temporary directory so we can mutate it.
|
||||
copy_dir_all(
|
||||
context
|
||||
.workspace_root
|
||||
|
|
@ -2216,7 +2217,7 @@ fn sync_editable() -> Result<()> {
|
|||
poetry_editable = poetry_editable.display()
|
||||
})?;
|
||||
|
||||
// Install the editable packages.
|
||||
// Install the editable package.
|
||||
uv_snapshot!(context.filters(), context.pip_sync()
|
||||
.arg(requirements_txt.path()), @r###"
|
||||
success: true
|
||||
|
|
@ -2233,7 +2234,20 @@ fn sync_editable() -> Result<()> {
|
|||
"###
|
||||
);
|
||||
|
||||
// Reinstall the editable packages.
|
||||
// Re-install the editable package. This is a no-op.
|
||||
uv_snapshot!(context.filters(), context.pip_sync()
|
||||
.arg(requirements_txt.path()), @r###"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
----- stderr -----
|
||||
Resolved 3 packages in [TIME]
|
||||
Audited 3 packages in [TIME]
|
||||
"###
|
||||
);
|
||||
|
||||
// Reinstall the editable package. This won't trigger a rebuild, but it will trigger an install.
|
||||
uv_snapshot!(context.filters(), context.pip_sync()
|
||||
.arg(requirements_txt.path())
|
||||
.arg("--reinstall-package")
|
||||
|
|
@ -2244,7 +2258,6 @@ fn sync_editable() -> Result<()> {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 3 packages in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- poetry-editable==0.1.0 (from file://[TEMP_DIR]/poetry_editable)
|
||||
|
|
@ -2260,7 +2273,7 @@ fn sync_editable() -> Result<()> {
|
|||
"#};
|
||||
context.assert_command(check_installed).success();
|
||||
|
||||
// Edit the sources and make sure the changes are respected without syncing again
|
||||
// Edit the sources and make sure the changes are respected without syncing again.
|
||||
let python_version_1 = indoc::indoc! {r"
|
||||
version = 1
|
||||
"};
|
||||
|
|
@ -2285,6 +2298,8 @@ fn sync_editable() -> Result<()> {
|
|||
"};
|
||||
context.assert_command(check_installed).success();
|
||||
|
||||
// Reinstall the editable package. This won't trigger a rebuild or reinstall, since we only
|
||||
// detect changes to metadata files (like `pyproject.toml`).
|
||||
uv_snapshot!(context.filters(), context.pip_sync()
|
||||
.arg(requirements_txt.path()), @r###"
|
||||
success: true
|
||||
|
|
@ -2293,7 +2308,35 @@ fn sync_editable() -> Result<()> {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 3 packages in [TIME]
|
||||
Audited 3 packages in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- poetry-editable==0.1.0 (from file://[TEMP_DIR]/poetry_editable)
|
||||
+ poetry-editable==0.1.0 (from file://[TEMP_DIR]/poetry_editable)
|
||||
"###
|
||||
);
|
||||
|
||||
// Modify the `pyproject.toml` file.
|
||||
let pyproject_toml = poetry_editable.path().join("pyproject.toml");
|
||||
let pyproject_toml_contents = fs_err::read_to_string(&pyproject_toml)?;
|
||||
fs_err::write(
|
||||
&pyproject_toml,
|
||||
pyproject_toml_contents.replace("0.1.0", "0.1.1"),
|
||||
)?;
|
||||
|
||||
// Reinstall the editable package. This will trigger a rebuild and reinstall.
|
||||
uv_snapshot!(context.filters(), context.pip_sync()
|
||||
.arg(requirements_txt.path()), @r###"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
----- stderr -----
|
||||
Resolved 3 packages in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- poetry-editable==0.1.0 (from file://[TEMP_DIR]/poetry_editable)
|
||||
+ poetry-editable==0.1.1 (from file://[TEMP_DIR]/poetry_editable)
|
||||
"###
|
||||
);
|
||||
|
||||
|
|
@ -2486,7 +2529,6 @@ fn sync_editable_and_local() -> Result<()> {
|
|||
|
||||
----- stderr -----
|
||||
Resolved 1 package in [TIME]
|
||||
Prepared 1 package in [TIME]
|
||||
Uninstalled 1 package in [TIME]
|
||||
Installed 1 package in [TIME]
|
||||
- black==0.1.0 (from file://[TEMP_DIR]/black_editable)
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ fn run_with_python_version() -> Result<()> {
|
|||
Removed virtual environment at: .venv
|
||||
Creating virtualenv at: .venv
|
||||
Resolved 5 packages in [TIME]
|
||||
Prepared 4 packages in [TIME]
|
||||
Prepared 3 packages in [TIME]
|
||||
Installed 4 packages in [TIME]
|
||||
+ anyio==3.6.0
|
||||
+ foo==1.0.0 (from file://[TEMP_DIR]/)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue