Make uv cache clean parallel process safe (#15888)
Some checks are pending
CI / integration test | pypy on windows (push) Blocked by required conditions
CI / Determine changes (push) Waiting to run
CI / cargo clippy | ubuntu (push) Blocked by required conditions
CI / cargo clippy | windows (push) Blocked by required conditions
CI / cargo dev generate-all (push) Blocked by required conditions
CI / cargo shear (push) Waiting to run
CI / cargo test | ubuntu (push) Blocked by required conditions
CI / cargo test | macos (push) Blocked by required conditions
CI / smoke test | macos (push) Blocked by required conditions
CI / build binary | windows x86_64 (push) Blocked by required conditions
CI / build binary | msrv (push) Blocked by required conditions
CI / ecosystem test | prefecthq/prefect (push) Blocked by required conditions
CI / ecosystem test | pallets/flask (push) Blocked by required conditions
CI / smoke test | linux (push) Blocked by required conditions
CI / smoke test | linux aarch64 (push) Blocked by required conditions
CI / check system | alpine (push) Blocked by required conditions
CI / smoke test | windows x86_64 (push) Blocked by required conditions
CI / smoke test | windows aarch64 (push) Blocked by required conditions
CI / integration test | activate nushell venv (push) Blocked by required conditions
CI / integration test | conda on ubuntu (push) Blocked by required conditions
CI / integration test | deadsnakes python3.9 on ubuntu (push) Blocked by required conditions
CI / lint (push) Waiting to run
CI / cargo test | windows (push) Blocked by required conditions
CI / check windows trampoline | aarch64 (push) Blocked by required conditions
CI / check windows trampoline | i686 (push) Blocked by required conditions
CI / check windows trampoline | x86_64 (push) Blocked by required conditions
CI / test windows trampoline | aarch64 (push) Blocked by required conditions
CI / test windows trampoline | i686 (push) Blocked by required conditions
CI / test windows trampoline | x86_64 (push) Blocked by required conditions
CI / typos (push) Waiting to run
CI / mkdocs (push) Waiting to run
CI / build binary | linux libc (push) Blocked by required conditions
CI / build binary | linux aarch64 (push) Blocked by required conditions
CI / build binary | linux musl (push) Blocked by required conditions
CI / build binary | macos aarch64 (push) Blocked by required conditions
CI / build binary | macos x86_64 (push) Blocked by required conditions
CI / build binary | windows aarch64 (push) Blocked by required conditions
CI / build binary | freebsd (push) Blocked by required conditions
CI / ecosystem test | pydantic/pydantic-core (push) Blocked by required conditions
CI / check system | aarch64 python3.13 on windows aarch64 (push) Blocked by required conditions
CI / check system | conda3.11 on linux x86-64 (push) Blocked by required conditions
CI / integration test | free-threaded on windows (push) Blocked by required conditions
CI / integration test | aarch64 windows implicit (push) Blocked by required conditions
CI / integration test | aarch64 windows explicit (push) Blocked by required conditions
CI / integration test | pypy on ubuntu (push) Blocked by required conditions
CI / integration test | graalpy on ubuntu (push) Blocked by required conditions
CI / integration test | graalpy on windows (push) Blocked by required conditions
CI / integration test | pyodide on ubuntu (push) Blocked by required conditions
CI / integration test | github actions (push) Blocked by required conditions
CI / integration test | free-threaded python on github actions (push) Blocked by required conditions
CI / integration test | pyenv on wsl x86-64 (push) Blocked by required conditions
CI / integration test | determine publish changes (push) Blocked by required conditions
CI / integration test | registries (push) Blocked by required conditions
CI / integration test | uv publish (push) Blocked by required conditions
CI / integration test | uv_build (push) Blocked by required conditions
CI / check cache | ubuntu (push) Blocked by required conditions
CI / check cache | macos aarch64 (push) Blocked by required conditions
CI / check system | python on debian (push) Blocked by required conditions
CI / check system | python on fedora (push) Blocked by required conditions
CI / check system | python on ubuntu (push) Blocked by required conditions
CI / check system | python on rocky linux 8 (push) Blocked by required conditions
CI / check system | python on rocky linux 9 (push) Blocked by required conditions
CI / check system | graalpy on ubuntu (push) Blocked by required conditions
CI / check system | pypy on ubuntu (push) Blocked by required conditions
CI / check system | pyston (push) Blocked by required conditions
CI / check system | python on macos aarch64 (push) Blocked by required conditions
CI / check system | homebrew python on macos aarch64 (push) Blocked by required conditions
CI / check system | x86-64 python on macos aarch64 (push) Blocked by required conditions
CI / check system | python on macos x86-64 (push) Blocked by required conditions
CI / check system | python3.10 on windows x86-64 (push) Blocked by required conditions
CI / check system | python3.10 on windows x86 (push) Blocked by required conditions
CI / check system | python3.13 on windows x86-64 (push) Blocked by required conditions
CI / check system | x86-64 python3.13 on windows aarch64 (push) Blocked by required conditions
CI / check system | windows registry (push) Blocked by required conditions
CI / check system | python3.12 via chocolatey (push) Blocked by required conditions
CI / check system | python3.9 via pyenv (push) Blocked by required conditions
CI / check system | python3.13 (push) Blocked by required conditions
CI / check system | conda3.11 on macos aarch64 (push) Blocked by required conditions
CI / check system | conda3.8 on macos aarch64 (push) Blocked by required conditions
CI / check system | conda3.8 on linux x86-64 (push) Blocked by required conditions
CI / check system | conda3.11 on windows x86-64 (push) Blocked by required conditions
CI / check system | conda3.8 on windows x86-64 (push) Blocked by required conditions
CI / check system | embedded python3.10 on windows x86-64 (push) Blocked by required conditions
CI / benchmarks | walltime aarch64 linux (push) Blocked by required conditions
CI / benchmarks | instrumented (push) Blocked by required conditions
zizmor / Run zizmor (push) Waiting to run
CI / check system | amazonlinux (push) Blocked by required conditions

Currently, `uv cache clean` and `uv cache prune` can cause crashes in
other uv processes running in parallel by removing their in-use files.

We can solve this by using a shared (read) lock on the cache directory,
while the `uv cache` operations use an exclusive (write) lock. The
drawback is that this is always one extra lock, and that we assume that
all platforms support shared locks.

Once Rust 1.89 fulfills our N-2 policy, we can add support for these
methods in fs_err and switch to
https://doc.rust-lang.org/std/fs/struct.File.html#platform-specific-behavior-2.

**Test Plan**

Open one terminal, run:

```
uv venv -c -p 3.13
UV_CACHE_DIR=cache uv cache clean
UV_CACHE_DIR=cache uv pip install numpy==2.0.0
```

Open another terminal, run:

```
UV_CACHE_DIR=cache uv cache clean
```

Fixes #15704
Part of #13883
This commit is contained in:
konsti 2025-09-19 10:21:22 +02:00 committed by GitHub
parent 0889d53c25
commit 00aa2ab672
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 236 additions and 51 deletions

View file

@ -693,9 +693,50 @@ impl LockedFile {
}
}
/// The same as [`LockedFile::acquire`], but for synchronous contexts. Do not use from an async
/// context, as this can block the runtime while waiting for another process to release the
/// lock.
/// Inner implementation for [`LockedFile::acquire_shared_blocking`] and
/// [`LockedFile::acquire_blocking`].
fn lock_file_shared_blocking(
file: fs_err::File,
resource: &str,
) -> Result<Self, std::io::Error> {
trace!(
"Checking shared lock for `{resource}` at `{}`",
file.path().user_display()
);
// TODO(konsti): Update fs_err to support this.
match FileExt::try_lock_shared(file.file()) {
Ok(()) => {
debug!("Acquired shared lock for `{resource}`");
Ok(Self(file))
}
Err(err) => {
// Log error code and enum kind to help debugging more exotic failures.
if err.kind() != std::io::ErrorKind::WouldBlock {
debug!("Try lock error: {err:?}");
}
info!(
"Waiting to acquire shared lock for `{resource}` at `{}`",
file.path().user_display(),
);
FileExt::lock_shared(file.file()).map_err(|err| {
// Not an fs_err method, we need to build our own path context
std::io::Error::other(format!(
"Could not acquire shared lock for `{resource}` at `{}`: {}",
file.path().user_display(),
err
))
})?;
debug!("Acquired shared lock for `{resource}`");
Ok(Self(file))
}
}
}
/// The same as [`LockedFile::acquire`], but for synchronous contexts.
///
/// Do not use from an async context, as this can block the runtime while waiting for another
/// process to release the lock.
pub fn acquire_blocking(
path: impl AsRef<Path>,
resource: impl Display,
@ -705,6 +746,19 @@ impl LockedFile {
Self::lock_file_blocking(file, &resource)
}
/// The same as [`LockedFile::acquire_blocking`], but for synchronous contexts.
///
/// Do not use from an async context, as this can block the runtime while waiting for another
/// process to release the lock.
pub fn acquire_shared_blocking(
path: impl AsRef<Path>,
resource: impl Display,
) -> Result<Self, std::io::Error> {
let file = Self::create(path)?;
let resource = resource.to_string();
Self::lock_file_shared_blocking(file, &resource)
}
/// Acquire a cross-process lock for a resource using a file at the provided path.
#[cfg(feature = "tokio")]
pub async fn acquire(
@ -716,6 +770,18 @@ impl LockedFile {
tokio::task::spawn_blocking(move || Self::lock_file_blocking(file, &resource)).await?
}
/// Acquire a cross-process read lock for a shared resource using a file at the provided path.
#[cfg(feature = "tokio")]
pub async fn acquire_shared(
path: impl AsRef<Path>,
resource: impl Display,
) -> Result<Self, std::io::Error> {
let file = Self::create(path)?;
let resource = resource.to_string();
tokio::task::spawn_blocking(move || Self::lock_file_shared_blocking(file, &resource))
.await?
}
#[cfg(unix)]
fn create(path: impl AsRef<Path>) -> Result<fs_err::File, std::io::Error> {
use std::os::unix::fs::PermissionsExt;