From 93d444cca4a9a5b3ec428865061c1f30624efe8c Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 2 Jul 2025 15:14:01 -0400 Subject: [PATCH 1/2] Create a dedicated crate --- Cargo.lock | 20 +++ Cargo.toml | 1 + crates/uv-build-frontend/Cargo.toml | 1 + crates/uv-build-frontend/src/lib.rs | 2 +- crates/uv-cache/Cargo.toml | 1 + crates/uv-cache/src/lib.rs | 3 +- crates/uv-fs/src/lib.rs | 135 +---------------- crates/uv-git/Cargo.toml | 1 + crates/uv-git/src/resolver.rs | 2 +- crates/uv-lock/Cargo.toml | 31 ++++ crates/uv-lock/src/lib.rs | 208 ++++++++++++++++++++++++++ crates/uv-python/Cargo.toml | 1 + crates/uv-python/src/environment.rs | 3 +- crates/uv-python/src/interpreter.rs | 3 +- crates/uv-python/src/managed.rs | 3 +- crates/uv-state/src/lib.rs | 3 + crates/uv-static/src/env_vars.rs | 3 + crates/uv-tool/Cargo.toml | 1 + crates/uv-tool/src/lib.rs | 3 +- crates/uv/Cargo.toml | 1 + crates/uv/src/commands/project/add.rs | 3 +- crates/uv/src/commands/project/mod.rs | 3 +- 22 files changed, 289 insertions(+), 143 deletions(-) create mode 100644 crates/uv-lock/Cargo.toml create mode 100644 crates/uv-lock/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index f9e51c47a..0d1654dc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4676,6 +4676,7 @@ dependencies = [ "uv-git-types", "uv-install-wheel", "uv-installer", + "uv-lock", "uv-normalize", "uv-pep440", "uv-pep508", @@ -4835,6 +4836,7 @@ dependencies = [ "uv-distribution", "uv-distribution-types", "uv-fs", + "uv-lock", "uv-pep440", "uv-pep508", "uv-pypi-types", @@ -4864,6 +4866,7 @@ dependencies = [ "uv-dirs", "uv-distribution-types", "uv-fs", + "uv-lock", "uv-normalize", "uv-pypi-types", "uv-redacted", @@ -5282,6 +5285,7 @@ dependencies = [ "uv-cache-key", "uv-fs", "uv-git-types", + "uv-lock", "uv-redacted", "uv-static", "uv-version", @@ -5392,6 +5396,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "uv-lock" +version = "0.0.1" +dependencies = [ + "fs-err 3.1.1", + "fs2", + "tempfile", + "tokio", + "tracing", + "uv-fs", + "uv-state", + "uv-static", +] + [[package]] name = "uv-macros" version = "0.0.1" @@ -5625,6 +5643,7 @@ dependencies = [ "uv-extract", "uv-fs", "uv-install-wheel", + "uv-lock", "uv-pep440", "uv-pep508", "uv-platform-tags", @@ -5889,6 +5908,7 @@ dependencies = [ "uv-fs", "uv-install-wheel", "uv-installer", + "uv-lock", "uv-pep440", "uv-pep508", "uv-pypi-types", diff --git a/Cargo.toml b/Cargo.toml index 817c5c62b..e98cdd4dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ uv-git-types = { path = "crates/uv-git-types" } uv-globfilter = { path = "crates/uv-globfilter" } uv-install-wheel = { path = "crates/uv-install-wheel", default-features = false } uv-installer = { path = "crates/uv-installer" } +uv-lock = { path = "crates/uv-lock" } uv-macros = { path = "crates/uv-macros" } uv-metadata = { path = "crates/uv-metadata" } uv-normalize = { path = "crates/uv-normalize" } diff --git a/crates/uv-build-frontend/Cargo.toml b/crates/uv-build-frontend/Cargo.toml index 748e7bb28..b47abef42 100644 --- a/crates/uv-build-frontend/Cargo.toml +++ b/crates/uv-build-frontend/Cargo.toml @@ -22,6 +22,7 @@ uv-configuration = { workspace = true } uv-distribution = { workspace = true } uv-distribution-types = { workspace = true } uv-fs = { workspace = true } +uv-lock = { workspace = true, features = ["tokio"] } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } uv-pypi-types = { workspace = true } diff --git a/crates/uv-build-frontend/src/lib.rs b/crates/uv-build-frontend/src/lib.rs index 5cbaece2e..1d3f02e74 100644 --- a/crates/uv-build-frontend/src/lib.rs +++ b/crates/uv-build-frontend/src/lib.rs @@ -32,8 +32,8 @@ use uv_configuration::PreviewMode; use uv_configuration::{BuildKind, BuildOutput, ConfigSettings, SourceStrategy}; use uv_distribution::BuildRequires; use uv_distribution_types::{IndexLocations, Requirement, Resolution}; -use uv_fs::LockedFile; use uv_fs::{PythonExt, Simplified}; +use uv_lock::LockedFile; use uv_pep440::Version; use uv_pep508::PackageName; use uv_pypi_types::VerbatimParsedUrl; diff --git a/crates/uv-cache/Cargo.toml b/crates/uv-cache/Cargo.toml index 779309f0f..b2efc78c6 100644 --- a/crates/uv-cache/Cargo.toml +++ b/crates/uv-cache/Cargo.toml @@ -22,6 +22,7 @@ uv-cache-key = { workspace = true } uv-dirs = { workspace = true } uv-distribution-types = { workspace = true } uv-fs = { workspace = true, features = ["tokio"] } +uv-lock = { workspace = true, features = ["tokio"] } uv-normalize = { workspace = true } uv-pypi-types = { workspace = true } uv-redacted = { workspace = true } diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index af28bb26c..f6db6b86d 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -11,7 +11,8 @@ use tracing::debug; pub use archive::ArchiveId; use uv_cache_info::Timestamp; -use uv_fs::{LockedFile, cachedir, directories}; +use uv_fs::{cachedir, directories}; +use uv_lock::LockedFile; use uv_normalize::PackageName; use uv_pypi_types::ResolutionMetadata; diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index dcc0f00b2..ec2ee81c2 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -1,10 +1,8 @@ -use std::fmt::Display; use std::io; use std::path::{Path, PathBuf}; -use fs2::FileExt; use tempfile::NamedTempFile; -use tracing::{debug, error, info, trace, warn}; +use tracing::warn; pub use crate::path::*; @@ -599,137 +597,6 @@ pub fn is_virtualenv_base(path: impl AsRef) -> bool { path.as_ref().join("pyvenv.cfg").is_file() } -/// A file lock that is automatically released when dropped. -#[derive(Debug)] -#[must_use] -pub struct LockedFile(fs_err::File); - -impl LockedFile { - /// Inner implementation for [`LockedFile::acquire_blocking`] and [`LockedFile::acquire`]. - fn lock_file_blocking(file: fs_err::File, resource: &str) -> Result { - trace!( - "Checking lock for `{resource}` at `{}`", - file.path().user_display() - ); - match file.file().try_lock_exclusive() { - Ok(()) => { - debug!("Acquired lock for `{resource}`"); - Ok(Self(file)) - } - Err(err) => { - // Log error code and enum kind to help debugging more exotic failures. - if err.kind() != std::io::ErrorKind::WouldBlock { - debug!("Try lock error: {err:?}"); - } - info!( - "Waiting to acquire lock for `{resource}` at `{}`", - file.path().user_display(), - ); - file.file().lock_exclusive().map_err(|err| { - // Not an fs_err method, we need to build our own path context - std::io::Error::other(format!( - "Could not acquire lock for `{resource}` at `{}`: {}", - file.path().user_display(), - err - )) - })?; - - debug!("Acquired lock for `{resource}`"); - Ok(Self(file)) - } - } - } - - /// The same as [`LockedFile::acquire`], but for synchronous contexts. Do not use from an async - /// context, as this can block the runtime while waiting for another process to release the - /// lock. - pub fn acquire_blocking( - path: impl AsRef, - resource: impl Display, - ) -> Result { - let file = Self::create(path)?; - let resource = resource.to_string(); - Self::lock_file_blocking(file, &resource) - } - - /// Acquire a cross-process lock for a resource using a file at the provided path. - #[cfg(feature = "tokio")] - pub async fn acquire( - path: impl AsRef, - resource: impl Display, - ) -> Result { - let file = Self::create(path)?; - let resource = resource.to_string(); - tokio::task::spawn_blocking(move || Self::lock_file_blocking(file, &resource)).await? - } - - #[cfg(unix)] - fn create(path: impl AsRef) -> Result { - use std::os::unix::fs::PermissionsExt; - - // If path already exists, return it. - if let Ok(file) = fs_err::OpenOptions::new() - .read(true) - .write(true) - .open(path.as_ref()) - { - return Ok(file); - } - - // Otherwise, create a temporary file with 777 permissions. We must set - // permissions _after_ creating the file, to override the `umask`. - let file = if let Some(parent) = path.as_ref().parent() { - NamedTempFile::new_in(parent)? - } else { - NamedTempFile::new()? - }; - if let Err(err) = file - .as_file() - .set_permissions(std::fs::Permissions::from_mode(0o777)) - { - warn!("Failed to set permissions on temporary file: {err}"); - } - - // Try to move the file to path, but if path exists now, just open path - match file.persist_noclobber(path.as_ref()) { - Ok(file) => Ok(fs_err::File::from_parts(file, path.as_ref())), - Err(err) => { - if err.error.kind() == std::io::ErrorKind::AlreadyExists { - fs_err::OpenOptions::new() - .read(true) - .write(true) - .open(path.as_ref()) - } else { - Err(err.error) - } - } - } - } - - #[cfg(not(unix))] - fn create(path: impl AsRef) -> std::io::Result { - fs_err::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(path.as_ref()) - } -} - -impl Drop for LockedFile { - fn drop(&mut self) { - if let Err(err) = fs2::FileExt::unlock(self.0.file()) { - error!( - "Failed to unlock {}; program may be stuck: {}", - self.0.path().display(), - err - ); - } else { - debug!("Released lock at `{}`", self.0.path().display()); - } - } -} - /// An asynchronous reader that reports progress as bytes are read. #[cfg(feature = "tokio")] pub struct ProgressReader { diff --git a/crates/uv-git/Cargo.toml b/crates/uv-git/Cargo.toml index 39c90849e..315519b0d 100644 --- a/crates/uv-git/Cargo.toml +++ b/crates/uv-git/Cargo.toml @@ -20,6 +20,7 @@ uv-auth = { workspace = true } uv-cache-key = { workspace = true } uv-fs = { workspace = true, features = ["tokio"] } uv-git-types = { workspace = true } +uv-lock = { workspace = true, features = ["tokio"] } uv-redacted = { workspace = true } uv-static = { workspace = true } uv-version = { workspace = true } diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs index 3c12fc589..823edd2c6 100644 --- a/crates/uv-git/src/resolver.rs +++ b/crates/uv-git/src/resolver.rs @@ -10,8 +10,8 @@ use reqwest_middleware::ClientWithMiddleware; use tracing::debug; use uv_cache_key::{RepositoryUrl, cache_digest}; -use uv_fs::LockedFile; use uv_git_types::{GitHubRepository, GitOid, GitReference, GitUrl}; +use uv_lock::LockedFile; use uv_static::EnvVars; use uv_version::version; diff --git a/crates/uv-lock/Cargo.toml b/crates/uv-lock/Cargo.toml new file mode 100644 index 000000000..2ac86abb0 --- /dev/null +++ b/crates/uv-lock/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "uv-lock" +version = "0.0.1" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +uv-fs = { workspace = true } +uv-state = { workspace = true } +uv-static = { workspace = true } + +fs-err = { workspace = true } +fs2 = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, optional = true} +tracing = { workspace = true } + +[features] +default = [] +tokio = ["dep:tokio", "fs-err/tokio"] diff --git a/crates/uv-lock/src/lib.rs b/crates/uv-lock/src/lib.rs new file mode 100644 index 000000000..ff4b798ea --- /dev/null +++ b/crates/uv-lock/src/lib.rs @@ -0,0 +1,208 @@ +use fs_err as fs; +use std::fmt::Display; +use std::io::Write; + +use std::path::{Path, PathBuf}; + +use fs2::FileExt; +use tempfile::NamedTempFile; +use tracing::{debug, error, info, trace, warn}; + +use uv_fs::Simplified; +use uv_state::{StateBucket, StateStore}; +use uv_static::EnvVars; + +/// Filesystem locks used to synchronize access to shared resources across processes. +#[derive(Debug, Clone)] +pub struct FilesystemLocks { + /// The path to the top-level directory of the filesystem locks. + root: PathBuf, +} + +impl FilesystemLocks { + /// A directory for filesystem locks at `root`. + fn from_path(root: impl Into) -> Self { + Self { root: root.into() } + } + + /// Create a new [`FilesystemLocks`] from settings. + /// + /// Prefer, in order: + /// + /// 1. The specific tool directory specified by the user, i.e., `UV_LOCK_DIR` + /// 2. A directory in the system-appropriate user-level data directory, e.g., `~/.local/uv/tools` + /// 3. A directory in the local data directory, e.g., `./.uv/tools` + pub fn from_settings() -> Result { + if let Some(lock_dir) = std::env::var_os(EnvVars::UV_LOCK_DIR).filter(|s| !s.is_empty()) { + Ok(Self::from_path(std::path::absolute(lock_dir)?)) + } else { + Ok(Self::from_path( + StateStore::from_settings(None)?.bucket(StateBucket::Locks), + )) + } + } + + /// Create a temporary directory. + pub fn temp() -> Result { + Ok(Self::from_path( + StateStore::temp()?.bucket(StateBucket::Locks), + )) + } + + /// Initialize the directory. + pub fn init(self) -> Result { + let root = &self.root; + + // Create the tools directory, if it doesn't exist. + fs::create_dir_all(root)?; + + // Add a .gitignore. + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join(".gitignore")) + { + Ok(mut file) => file.write_all(b"*")?, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => (), + Err(err) => return Err(err), + } + + Ok(self) + } + + /// Return the path of the directory. + pub fn root(&self) -> &Path { + &self.root + } +} + +/// A file lock that is automatically released when dropped. +#[derive(Debug)] +#[must_use] +pub struct LockedFile(fs_err::File); + +impl LockedFile { + /// Inner implementation for [`LockedFile::acquire_blocking`] and [`LockedFile::acquire`]. + fn lock_file_blocking(file: fs_err::File, resource: &str) -> Result { + trace!( + "Checking lock for `{resource}` at `{}`", + file.path().user_display() + ); + match file.file().try_lock_exclusive() { + Ok(()) => { + debug!("Acquired lock for `{resource}`"); + Ok(Self(file)) + } + Err(err) => { + // Log error code and enum kind to help debugging more exotic failures. + if err.kind() != std::io::ErrorKind::WouldBlock { + debug!("Try lock error: {err:?}"); + } + info!( + "Waiting to acquire lock for `{resource}` at `{}`", + file.path().user_display(), + ); + file.file().lock_exclusive().map_err(|err| { + // Not an fs_err method, we need to build our own path context + std::io::Error::other(format!( + "Could not acquire lock for `{resource}` at `{}`: {}", + file.path().user_display(), + err + )) + })?; + + debug!("Acquired lock for `{resource}`"); + Ok(Self(file)) + } + } + } + + /// The same as [`LockedFile::acquire`], but for synchronous contexts. Do not use from an async + /// context, as this can block the runtime while waiting for another process to release the + /// lock. + pub fn acquire_blocking( + path: impl AsRef, + resource: impl Display, + ) -> Result { + let file = Self::create(path)?; + let resource = resource.to_string(); + Self::lock_file_blocking(file, &resource) + } + + /// Acquire a cross-process lock for a resource using a file at the provided path. + #[cfg(feature = "tokio")] + pub async fn acquire( + path: impl AsRef, + resource: impl Display, + ) -> Result { + let file = Self::create(path)?; + let resource = resource.to_string(); + tokio::task::spawn_blocking(move || Self::lock_file_blocking(file, &resource)).await? + } + + #[cfg(unix)] + fn create(path: impl AsRef) -> Result { + use std::os::unix::fs::PermissionsExt; + + // If path already exists, return it. + if let Ok(file) = fs_err::OpenOptions::new() + .read(true) + .write(true) + .open(path.as_ref()) + { + return Ok(file); + } + + // Otherwise, create a temporary file with 777 permissions. We must set + // permissions _after_ creating the file, to override the `umask`. + let file = if let Some(parent) = path.as_ref().parent() { + NamedTempFile::new_in(parent)? + } else { + NamedTempFile::new()? + }; + if let Err(err) = file + .as_file() + .set_permissions(std::fs::Permissions::from_mode(0o777)) + { + warn!("Failed to set permissions on temporary file: {err}"); + } + + // Try to move the file to path, but if path exists now, just open path + match file.persist_noclobber(path.as_ref()) { + Ok(file) => Ok(fs_err::File::from_parts(file, path.as_ref())), + Err(err) => { + if err.error.kind() == std::io::ErrorKind::AlreadyExists { + fs_err::OpenOptions::new() + .read(true) + .write(true) + .open(path.as_ref()) + } else { + Err(err.error) + } + } + } + } + + #[cfg(not(unix))] + fn create(path: impl AsRef) -> std::io::Result { + fs_err::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path.as_ref()) + } +} + +impl Drop for LockedFile { + fn drop(&mut self) { + if let Err(err) = fs2::FileExt::unlock(self.0.file()) { + error!( + "Failed to unlock {}; program may be stuck: {}", + self.0.path().display(), + err + ); + } else { + debug!("Released lock at `{}`", self.0.path().display()); + } + } +} diff --git a/crates/uv-python/Cargo.toml b/crates/uv-python/Cargo.toml index d008b2d4e..4caa1bac6 100644 --- a/crates/uv-python/Cargo.toml +++ b/crates/uv-python/Cargo.toml @@ -26,6 +26,7 @@ uv-distribution-filename = { workspace = true } uv-extract = { workspace = true } uv-fs = { workspace = true } uv-install-wheel = { workspace = true } +uv-lock = { workspace = true, features = ["tokio"] } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } uv-platform-tags = { workspace = true } diff --git a/crates/uv-python/src/environment.rs b/crates/uv-python/src/environment.rs index 02f9fd683..65e69d5cf 100644 --- a/crates/uv-python/src/environment.rs +++ b/crates/uv-python/src/environment.rs @@ -8,7 +8,8 @@ use tracing::debug; use uv_cache::Cache; use uv_configuration::PreviewMode; -use uv_fs::{LockedFile, Simplified}; +use uv_fs::Simplified; +use uv_lock::LockedFile; use uv_pep440::Version; use crate::discovery::find_python_installation; diff --git a/crates/uv-python/src/interpreter.rs b/crates/uv-python/src/interpreter.rs index 0f074ebb6..73c4a1812 100644 --- a/crates/uv-python/src/interpreter.rs +++ b/crates/uv-python/src/interpreter.rs @@ -17,8 +17,9 @@ use tracing::{debug, trace, warn}; use uv_cache::{Cache, CacheBucket, CachedByTimestamp, Freshness}; use uv_cache_info::Timestamp; use uv_cache_key::cache_digest; -use uv_fs::{LockedFile, PythonExt, Simplified, write_atomic_sync}; +use uv_fs::{PythonExt, Simplified, write_atomic_sync}; use uv_install_wheel::Layout; +use uv_lock::LockedFile; use uv_pep440::Version; use uv_pep508::{MarkerEnvironment, StringVersion}; use uv_platform_tags::Platform; diff --git a/crates/uv-python/src/managed.rs b/crates/uv-python/src/managed.rs index ad1dacac6..f7640f60a 100644 --- a/crates/uv-python/src/managed.rs +++ b/crates/uv-python/src/managed.rs @@ -16,7 +16,8 @@ use uv_configuration::PreviewMode; #[cfg(windows)] use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT; -use uv_fs::{LockedFile, Simplified, replace_symlink, symlink_or_copy_file}; +use uv_fs::{Simplified, replace_symlink, symlink_or_copy_file}; +use uv_lock::LockedFile; use uv_state::{StateBucket, StateStore}; use uv_static::EnvVars; use uv_trampoline_builder::{Launcher, windows_python_launcher}; diff --git a/crates/uv-state/src/lib.rs b/crates/uv-state/src/lib.rs index 2fd663b7f..201141cd0 100644 --- a/crates/uv-state/src/lib.rs +++ b/crates/uv-state/src/lib.rs @@ -105,6 +105,8 @@ pub enum StateBucket { ManagedPython, /// Installed tools. Tools, + /// File-system locks. + Locks, } impl StateBucket { @@ -112,6 +114,7 @@ impl StateBucket { match self { Self::ManagedPython => "python", Self::Tools => "tools", + Self::Locks => "locks", } } } diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index 4ac2976d9..8559756e5 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -247,6 +247,9 @@ impl EnvVars { /// Specifies the directory where uv stores managed tools. pub const UV_TOOL_DIR: &'static str = "UV_TOOL_DIR"; + /// Specifies the directory where uv stores filesystem locks. + pub const UV_LOCK_DIR: &'static str = "UV_LOCK_DIR"; + /// Specifies the "bin" directory for installing tool executables. pub const UV_TOOL_BIN_DIR: &'static str = "UV_TOOL_BIN_DIR"; diff --git a/crates/uv-tool/Cargo.toml b/crates/uv-tool/Cargo.toml index 210c17c00..cb254a71f 100644 --- a/crates/uv-tool/Cargo.toml +++ b/crates/uv-tool/Cargo.toml @@ -23,6 +23,7 @@ uv-distribution-types = { workspace = true } uv-fs = { workspace = true } uv-install-wheel = { workspace = true } uv-installer = { workspace = true } +uv-lock = { workspace = true, features = ["tokio"] } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } uv-pypi-types = { workspace = true } diff --git a/crates/uv-tool/src/lib.rs b/crates/uv-tool/src/lib.rs index ee80a2854..71974cfc4 100644 --- a/crates/uv-tool/src/lib.rs +++ b/crates/uv-tool/src/lib.rs @@ -19,8 +19,9 @@ use uv_install_wheel::read_record_file; pub use receipt::ToolReceipt; pub use tool::{Tool, ToolEntrypoint}; use uv_cache::Cache; -use uv_fs::{LockedFile, Simplified}; +use uv_fs::Simplified; use uv_installer::SitePackages; +use uv_lock::LockedFile; use uv_python::{Interpreter, PythonEnvironment}; use uv_state::{StateBucket, StateStore}; use uv_static::EnvVars; diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index 0a352d2b1..763ea54e5 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -35,6 +35,7 @@ uv-git = { workspace = true } uv-git-types = { workspace = true } uv-install-wheel = { workspace = true, default-features = false } uv-installer = { workspace = true } +uv-lock = { workspace = true, features = ["tokio"] } uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index 04fd7d822..7c09ddfd4 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -27,9 +27,10 @@ use uv_distribution_types::{ Index, IndexName, IndexUrl, IndexUrls, NameRequirementSpecification, Requirement, RequirementSource, UnresolvedRequirement, VersionId, }; -use uv_fs::{LockedFile, Simplified}; +use uv_fs::Simplified; use uv_git::GIT_STORE; use uv_git_types::GitReference; +use uv_lock::LockedFile; use uv_normalize::{DEV_DEPENDENCIES, DefaultExtras, DefaultGroups, PackageName}; use uv_pep508::{ExtraName, MarkerTree, UnnamedRequirement, VersionOrUrl}; use uv_pypi_types::{ParsedUrl, VerbatimParsedUrl}; diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index c327e8a44..8a0a8222c 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -21,9 +21,10 @@ use uv_distribution_types::{ Index, Requirement, RequiresPython, Resolution, UnresolvedRequirement, UnresolvedRequirementSpecification, }; -use uv_fs::{CWD, LockedFile, Simplified}; +use uv_fs::{CWD, Simplified}; use uv_git::ResolvedRepositoryReference; use uv_installer::{SatisfiesResult, SitePackages}; +use uv_lock::LockedFile; use uv_normalize::{DEV_DEPENDENCIES, DefaultGroups, ExtraName, GroupName, PackageName}; use uv_pep440::{TildeVersionSpecifier, Version, VersionSpecifiers}; use uv_pep508::MarkerTreeContents; From c3ed8b446706797c0fea6b54281e95af6e1ff7c6 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 2 Jul 2025 15:41:00 -0400 Subject: [PATCH 2/2] Use singleton --- Cargo.lock | 3 +- crates/uv-build-frontend/Cargo.toml | 1 - crates/uv-build-frontend/src/lib.rs | 8 +-- crates/uv-cache/src/lib.rs | 23 +++++-- crates/uv-fs/Cargo.toml | 1 - crates/uv-git/src/resolver.rs | 12 +--- crates/uv-lock/Cargo.toml | 1 + crates/uv-lock/src/lib.rs | 99 ++++++++++++++++----------- crates/uv-python/src/interpreter.rs | 23 +++---- crates/uv-python/src/managed.rs | 2 +- crates/uv-tool/src/lib.rs | 2 +- crates/uv/src/commands/project/mod.rs | 33 ++------- crates/uv/tests/it/common/mod.rs | 1 + crates/uv/tests/it/sync.rs | 12 ++-- docs/reference/environment.md | 4 ++ 15 files changed, 107 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d1654dc8..a47c0cfbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4831,7 +4831,6 @@ dependencies = [ "tokio", "toml_edit", "tracing", - "uv-cache-key", "uv-configuration", "uv-distribution", "uv-distribution-types", @@ -5252,7 +5251,6 @@ dependencies = [ "either", "encoding_rs_io", "fs-err 3.1.1", - "fs2", "junction", "path-slash", "percent-encoding", @@ -5405,6 +5403,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "uv-cache-key", "uv-fs", "uv-state", "uv-static", diff --git a/crates/uv-build-frontend/Cargo.toml b/crates/uv-build-frontend/Cargo.toml index b47abef42..765bea4b5 100644 --- a/crates/uv-build-frontend/Cargo.toml +++ b/crates/uv-build-frontend/Cargo.toml @@ -17,7 +17,6 @@ doctest = false workspace = true [dependencies] -uv-cache-key = { workspace = true } uv-configuration = { workspace = true } uv-distribution = { workspace = true } uv-distribution-types = { workspace = true } diff --git a/crates/uv-build-frontend/src/lib.rs b/crates/uv-build-frontend/src/lib.rs index 1d3f02e74..83c35a10c 100644 --- a/crates/uv-build-frontend/src/lib.rs +++ b/crates/uv-build-frontend/src/lib.rs @@ -27,7 +27,6 @@ use tokio::process::Command; use tokio::sync::{Mutex, Semaphore}; use tracing::{Instrument, debug, info_span, instrument, warn}; -use uv_cache_key::cache_digest; use uv_configuration::PreviewMode; use uv_configuration::{BuildKind, BuildOutput, ConfigSettings, SourceStrategy}; use uv_distribution::BuildRequires; @@ -451,12 +450,7 @@ impl SourceBuild { let mut source_tree_lock = None; if self.pep517_backend.is_setuptools() { debug!("Locking the source tree for setuptools"); - let canonical_source_path = self.source_tree.canonicalize()?; - let lock_path = env::temp_dir().join(format!( - "uv-setuptools-{}.lock", - cache_digest(&canonical_source_path) - )); - source_tree_lock = LockedFile::acquire(lock_path, self.source_tree.to_string_lossy()) + source_tree_lock = uv_lock::acquire_path(&self.source_tree) .await .inspect_err(|err| { warn!("Failed to acquire build lock: {err}"); diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index f6db6b86d..14362cdc0 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -11,6 +11,7 @@ use tracing::debug; pub use archive::ArchiveId; use uv_cache_info::Timestamp; +use uv_cache_key::{CacheKey, CacheKeyHasher}; use uv_fs::{cachedir, directories}; use uv_lock::LockedFile; use uv_normalize::PackageName; @@ -82,8 +83,7 @@ impl CacheEntry { /// Acquire the [`CacheEntry`] as an exclusive lock. pub async fn lock(&self) -> Result { - fs_err::create_dir_all(self.dir())?; - LockedFile::acquire(self.path(), self.path().display()).await + uv_lock::acquire_path(self.path()).await } } @@ -109,16 +109,27 @@ impl CacheShard { Self(self.0.join(dir.as_ref())) } - /// Acquire the cache entry as an exclusive lock. - pub async fn lock(&self) -> Result { - fs_err::create_dir_all(self.as_ref())?; - LockedFile::acquire(self.join(".lock"), self.display()).await + /// Return the path to the [`CacheShard`]. + #[inline] + pub fn path(&self) -> &Path { + &self.0 } /// Return the [`CacheShard`] as a [`PathBuf`]. pub fn into_path_buf(self) -> PathBuf { self.0 } + + /// Acquire the cache entry as an exclusive lock. + pub async fn lock(&self) -> Result { + uv_lock::acquire_path(self.path()).await + } +} + +impl CacheKey for CacheShard { + fn cache_key(&self, state: &mut CacheKeyHasher) { + self.0.cache_key(state); + } } impl AsRef for CacheShard { diff --git a/crates/uv-fs/Cargo.toml b/crates/uv-fs/Cargo.toml index fba4910e6..66903bdb8 100644 --- a/crates/uv-fs/Cargo.toml +++ b/crates/uv-fs/Cargo.toml @@ -20,7 +20,6 @@ dunce = { workspace = true } either = { workspace = true } encoding_rs_io = { workspace = true } fs-err = { workspace = true } -fs2 = { workspace = true } path-slash = { workspace = true } percent-encoding = { workspace = true } same-file = { workspace = true } diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs index 823edd2c6..4683d3cd3 100644 --- a/crates/uv-git/src/resolver.rs +++ b/crates/uv-git/src/resolver.rs @@ -5,13 +5,11 @@ use std::sync::Arc; use dashmap::DashMap; use dashmap::mapref::one::Ref; -use fs_err::tokio as fs; use reqwest_middleware::ClientWithMiddleware; use tracing::debug; -use uv_cache_key::{RepositoryUrl, cache_digest}; +use uv_cache_key::RepositoryUrl; use uv_git_types::{GitHubRepository, GitOid, GitReference, GitUrl}; -use uv_lock::LockedFile; use uv_static::EnvVars; use uv_version::version; @@ -164,14 +162,8 @@ impl GitResolver { }; // Avoid races between different processes, too. - let lock_dir = cache.join("locks"); - fs::create_dir_all(&lock_dir).await?; let repository_url = RepositoryUrl::new(url.repository()); - let _lock = LockedFile::acquire( - lock_dir.join(cache_digest(&repository_url)), - &repository_url, - ) - .await?; + let _lock = uv_lock::acquire_resource(repository_url).await?; // Fetch the Git repository. let source = if let Some(reporter) = reporter { diff --git a/crates/uv-lock/Cargo.toml b/crates/uv-lock/Cargo.toml index 2ac86abb0..b719a2d26 100644 --- a/crates/uv-lock/Cargo.toml +++ b/crates/uv-lock/Cargo.toml @@ -19,6 +19,7 @@ workspace = true uv-fs = { workspace = true } uv-state = { workspace = true } uv-static = { workspace = true } +uv-cache-key = { workspace = true } fs-err = { workspace = true } fs2 = { workspace = true } diff --git a/crates/uv-lock/src/lib.rs b/crates/uv-lock/src/lib.rs index ff4b798ea..c4b2d59c1 100644 --- a/crates/uv-lock/src/lib.rs +++ b/crates/uv-lock/src/lib.rs @@ -1,60 +1,76 @@ -use fs_err as fs; use std::fmt::Display; use std::io::Write; - use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use fs_err as fs; use fs2::FileExt; -use tempfile::NamedTempFile; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, trace}; +use uv_cache_key::{CacheKey, cache_digest}; use uv_fs::Simplified; use uv_state::{StateBucket, StateStore}; use uv_static::EnvVars; +/// Acquire a cross-process lock for files at the provided path. +#[cfg(feature = "tokio")] +pub async fn acquire_path(path: impl AsRef) -> Result { + let locks = get_or_init_locks()?; + let path = path.as_ref(); + LockedFile::acquire(locks.join(cache_digest(&path)), path.display()).await +} + +/// Acquire a cross-process lock for an arbitrary hashable resource (like a URL). +#[cfg(feature = "tokio")] +pub async fn acquire_resource( + resource: T, +) -> Result { + let locks = get_or_init_locks()?; + LockedFile::acquire(locks.join(cache_digest(&resource)), resource).await +} + +/// Get or initialize the global filesystem locks. +fn get_or_init_locks() -> std::io::Result<&'static FilesystemLocks> { + static FILESYSTEM_LOCKS: OnceLock = OnceLock::new(); + + // Return the existing filesystem locks, if they are already initialized. + if let Some(locks) = FILESYSTEM_LOCKS.get() { + return Ok(locks); + } + + // Initialize the filesystem locks. + let locks = FilesystemLocks::init()?; + _ = FILESYSTEM_LOCKS.set(locks); + + Ok(FILESYSTEM_LOCKS.get().unwrap()) +} + /// Filesystem locks used to synchronize access to shared resources across processes. #[derive(Debug, Clone)] -pub struct FilesystemLocks { +struct FilesystemLocks { /// The path to the top-level directory of the filesystem locks. root: PathBuf, } impl FilesystemLocks { - /// A directory for filesystem locks at `root`. - fn from_path(root: impl Into) -> Self { - Self { root: root.into() } - } - - /// Create a new [`FilesystemLocks`] from settings. + /// Create a new [`FilesystemLocks`]. /// /// Prefer, in order: /// /// 1. The specific tool directory specified by the user, i.e., `UV_LOCK_DIR` - /// 2. A directory in the system-appropriate user-level data directory, e.g., `~/.local/uv/tools` - /// 3. A directory in the local data directory, e.g., `./.uv/tools` - pub fn from_settings() -> Result { - if let Some(lock_dir) = std::env::var_os(EnvVars::UV_LOCK_DIR).filter(|s| !s.is_empty()) { - Ok(Self::from_path(std::path::absolute(lock_dir)?)) + /// 2. A directory in the system-appropriate user-level data directory, e.g., `~/.local/uv/locks` + /// 3. A directory in the local data directory, e.g., `./.uv/locks` + fn init() -> Result { + let root = if let Some(lock_dir) = + std::env::var_os(EnvVars::UV_LOCK_DIR).filter(|s| !s.is_empty()) + { + std::path::absolute(lock_dir)? } else { - Ok(Self::from_path( - StateStore::from_settings(None)?.bucket(StateBucket::Locks), - )) - } - } + StateStore::from_settings(None)?.bucket(StateBucket::Locks) + }; - /// Create a temporary directory. - pub fn temp() -> Result { - Ok(Self::from_path( - StateStore::temp()?.bucket(StateBucket::Locks), - )) - } - - /// Initialize the directory. - pub fn init(self) -> Result { - let root = &self.root; - - // Create the tools directory, if it doesn't exist. - fs::create_dir_all(root)?; + // Create the directory, if it doesn't exist. + fs::create_dir_all(&root)?; // Add a .gitignore. match fs::OpenOptions::new() @@ -67,12 +83,12 @@ impl FilesystemLocks { Err(err) => return Err(err), } - Ok(self) + Ok(Self { root }) } - /// Return the path of the directory. - pub fn root(&self) -> &Path { - &self.root + /// Join a path to the root of the locks directory. + fn join(&self, path: impl AsRef) -> PathBuf { + self.root.join(path) } } @@ -120,7 +136,8 @@ impl LockedFile { /// The same as [`LockedFile::acquire`], but for synchronous contexts. Do not use from an async /// context, as this can block the runtime while waiting for another process to release the /// lock. - pub fn acquire_blocking( + #[allow(dead_code)] + fn acquire_blocking( path: impl AsRef, resource: impl Display, ) -> Result { @@ -131,7 +148,7 @@ impl LockedFile { /// Acquire a cross-process lock for a resource using a file at the provided path. #[cfg(feature = "tokio")] - pub async fn acquire( + async fn acquire( path: impl AsRef, resource: impl Display, ) -> Result { @@ -143,6 +160,8 @@ impl LockedFile { #[cfg(unix)] fn create(path: impl AsRef) -> Result { use std::os::unix::fs::PermissionsExt; + use tempfile::NamedTempFile; + use tracing::warn; // If path already exists, return it. if let Ok(file) = fs_err::OpenOptions::new() diff --git a/crates/uv-python/src/interpreter.rs b/crates/uv-python/src/interpreter.rs index 73c4a1812..cfea78a29 100644 --- a/crates/uv-python/src/interpreter.rs +++ b/crates/uv-python/src/interpreter.rs @@ -1,10 +1,10 @@ use std::borrow::Cow; use std::env::consts::ARCH; use std::fmt::{Display, Formatter}; +use std::io; use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus}; use std::sync::OnceLock; -use std::{env, io}; use configparser::ini::Ini; use fs_err as fs; @@ -610,27 +610,20 @@ impl Interpreter { /// Grab a file lock for the environment to prevent concurrent writes across processes. pub async fn lock(&self) -> Result { - if let Some(target) = self.target() { + let path = if let Some(target) = self.target() { // If we're installing into a `--target`, use a target-specific lockfile. - LockedFile::acquire(target.root().join(".lock"), target.root().user_display()).await + target.root() } else if let Some(prefix) = self.prefix() { // Likewise, if we're installing into a `--prefix`, use a prefix-specific lockfile. - LockedFile::acquire(prefix.root().join(".lock"), prefix.root().user_display()).await + prefix.root() } else if self.is_virtualenv() { // If the environment a virtualenv, use a virtualenv-specific lockfile. - LockedFile::acquire( - self.sys_prefix.join(".lock"), - self.sys_prefix.user_display(), - ) - .await + &self.sys_prefix } else { // Otherwise, use a global lockfile. - LockedFile::acquire( - env::temp_dir().join(format!("uv-{}.lock", cache_digest(&self.sys_executable))), - self.sys_prefix.user_display(), - ) - .await - } + &self.sys_executable + }; + uv_lock::acquire_path(path).await } } diff --git a/crates/uv-python/src/managed.rs b/crates/uv-python/src/managed.rs index f7640f60a..18ab9a43f 100644 --- a/crates/uv-python/src/managed.rs +++ b/crates/uv-python/src/managed.rs @@ -125,7 +125,7 @@ impl ManagedPythonInstallations { /// Grab a file lock for the managed Python distribution directory to prevent concurrent access /// across processes. pub async fn lock(&self) -> Result { - Ok(LockedFile::acquire(self.root.join(".lock"), self.root.user_display()).await?) + Ok(uv_lock::acquire_path(&self.root).await?) } /// Prefer, in order: diff --git a/crates/uv-tool/src/lib.rs b/crates/uv-tool/src/lib.rs index 71974cfc4..e4cde0b8b 100644 --- a/crates/uv-tool/src/lib.rs +++ b/crates/uv-tool/src/lib.rs @@ -145,7 +145,7 @@ impl InstalledTools { /// Grab a file lock for the tools directory to prevent concurrent access across processes. pub async fn lock(&self) -> Result { - Ok(LockedFile::acquire(self.root.join(".lock"), self.root.user_display()).await?) + Ok(uv_lock::acquire_path(&self.root).await?) } /// Add a receipt for a tool. diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index 8a0a8222c..b555bb102 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -745,27 +745,9 @@ impl ScriptInterpreter { /// Grab a file lock for the script to prevent concurrent writes across processes. pub(crate) async fn lock(script: Pep723ItemRef<'_>) -> Result { match script { - Pep723ItemRef::Script(script) => { - LockedFile::acquire( - std::env::temp_dir().join(format!("uv-{}.lock", cache_digest(&script.path))), - script.path.simplified_display(), - ) - .await - } - Pep723ItemRef::Remote(.., url) => { - LockedFile::acquire( - std::env::temp_dir().join(format!("uv-{}.lock", cache_digest(url))), - url.to_string(), - ) - .await - } - Pep723ItemRef::Stdin(metadata) => { - LockedFile::acquire( - std::env::temp_dir().join(format!("uv-{}.lock", cache_digest(&metadata.raw))), - "stdin".to_string(), - ) - .await - } + Pep723ItemRef::Script(script) => uv_lock::acquire_path(&script.path).await, + Pep723ItemRef::Remote(.., url) => uv_lock::acquire_resource(&url).await, + Pep723ItemRef::Stdin(..) => uv_lock::acquire_resource("stdin").await, } } } @@ -1013,14 +995,7 @@ impl ProjectInterpreter { /// Grab a file lock for the environment to prevent concurrent writes across processes. pub(crate) async fn lock(workspace: &Workspace) -> Result { - LockedFile::acquire( - std::env::temp_dir().join(format!( - "uv-{}.lock", - cache_digest(workspace.install_path()) - )), - workspace.install_path().simplified_display(), - ) - .await + uv_lock::acquire_path(workspace.install_path()).await } } diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index 4c411899c..071fa3e1e 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -741,6 +741,7 @@ impl TestContext { .env(EnvVars::UV_TEST_NO_CLI_PROGRESS, "1") .env_remove(EnvVars::UV_CACHE_DIR) .env_remove(EnvVars::UV_TOOL_BIN_DIR) + .env_remove(EnvVars::UV_LOCK_DIR) .env_remove(EnvVars::XDG_CONFIG_HOME) .env_remove(EnvVars::XDG_DATA_HOME) .current_dir(self.temp_dir.path()); diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index a97775d9f..804b08088 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -9997,6 +9997,7 @@ fn read_only() -> Result<()> { use std::os::unix::fs::PermissionsExt; let context = TestContext::new("3.12"); + let lock_dir = context.temp_dir.child("locks"); let pyproject_toml = context.temp_dir.child("pyproject.toml"); pyproject_toml.write_str( @@ -10009,7 +10010,7 @@ fn read_only() -> Result<()> { "#, )?; - uv_snapshot!(context.filters(), context.sync(), @r###" + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_LOCK_DIR, lock_dir.as_os_str()), @r###" success: true exit_code: 0 ----- stdout ----- @@ -10024,12 +10025,13 @@ fn read_only() -> Result<()> { assert!(context.temp_dir.child("uv.lock").exists()); // Remove the flock. - fs_err::remove_file(context.venv.child(".lock"))?; + fs_err::remove_dir_all(&lock_dir)?; + fs_err::create_dir_all(&lock_dir)?; - // Make the virtual environment read and execute (but not write). - fs_err::set_permissions(&context.venv, std::fs::Permissions::from_mode(0o555))?; + // Make the lock directory read and execute (but not write). + fs_err::set_permissions(&lock_dir, std::fs::Permissions::from_mode(0o555))?; - uv_snapshot!(context.filters(), context.sync(), @r" + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_LOCK_DIR, lock_dir.as_os_str()), @r" success: true exit_code: 0 ----- stdout ----- diff --git a/docs/reference/environment.md b/docs/reference/environment.md index 61889ddb3..63a188219 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -173,6 +173,10 @@ a link mode. Equivalent to the `--locked` command-line argument. If set, uv will assert that the `uv.lock` remains unchanged. +### `UV_LOCK_DIR` + +Specifies the directory where uv stores filesystem locks. + ### `UV_LOG_CONTEXT` Add additional context and structure to log messages.