Add relocatable installs to support concurrency-safe cached environments (#5509)

## Summary

The idea here is similar to what we do for wheels: we create the
`CachedEnvironment` in the `archive-v0` bucket, then symlink it to its
content-addressed location. This ensures that we can always recreate
these environments without concern for whether anyone else is accessing
them.

Part of the challenge here is that we want the virtual environments to
be relocatable, because we're now building them in one location but
persisting them in another. This requires that we write relative (rather
than absolute) paths to scripts and entrypoints. The main risk with
relocatable virtual environments is that the scripts and entrypoints
_themselves_ are not relocatable, because they use a relative shebang.
But that's fine for cached environments, which are never intended to
leave the cache.

Closes https://github.com/astral-sh/uv/issues/5503.

crates/uv-installer/src/installer.rs

@@ -1,8 +1,7 @@
-use std::convert;
-
 use anyhow::{Context, Error, Result};
 use install_wheel_rs::{linker::LinkMode, Layout};
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use std::convert;
 use tokio::sync::oneshot;
 use tracing::instrument;
 

crates/uv/src/commands/project/environment.rs

@@ -5,7 +5,6 @@ use distribution_types::Resolution;
 use uv_cache::{Cache, CacheBucket};
 use uv_client::Connectivity;
 use uv_configuration::{Concurrency, PreviewMode};
-use uv_fs::{LockedFile, Simplified};
 use uv_python::{Interpreter, PythonEnvironment};
 use uv_requirements::RequirementsSpecification;
 
@@ -86,56 +85,25 @@ impl CachedEnvironment {
         // Search in the content-addressed cache.
         let cache_entry = cache.entry(CacheBucket::Environments, interpreter_hash, resolution_hash);
 
-        // Lock at the interpreter level, to avoid concurrent modification across processes.
-        fs_err::tokio::create_dir_all(cache_entry.dir()).await?;
-        let _lock = LockedFile::acquire(
-            cache_entry.dir().join(".lock"),
-            cache_entry.dir().user_display(),
-        )?;
-
-        let ok = cache_entry.path().join(".ok");
-
         if settings.reinstall.is_none() {
-            // If the receipt exists, return the environment.
-            if ok.is_file() {
-                debug!(
-                    "Reusing cached environment at: `{}`",
-                    cache_entry.path().display()
-                );
-                return Ok(Self(PythonEnvironment::from_root(
-                    cache_entry.path(),
-                    cache,
-                )?));
-            }
-        } else {
-            // If the receipt exists, remove it.
-            match fs_err::tokio::remove_file(&ok).await {
-                Ok(()) => {
-                    debug!(
-                        "Removed receipt for environment at: `{}`",
-                        cache_entry.path().display()
-                    );
+            if let Ok(root) = fs_err::read_link(cache_entry.path()) {
+                if let Ok(environment) = PythonEnvironment::from_root(root, cache) {
+                    return Ok(Self(environment));
                 }
-                Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
-                Err(err) => return Err(err.into()),
             }
         }
 
-        debug!(
-            "Creating cached environment at: `{}`",
-            cache_entry.path().display()
-        );
-
+        // Create the environment in the cache, then relocate it to its content-addressed location.
+        let temp_dir = cache.environment()?;
         let venv = uv_virtualenv::create_venv(
-            cache_entry.path(),
+            temp_dir.path(),
             interpreter,
             uv_virtualenv::Prompt::None,
             false,
             false,
-            false,
+            true,
         )?;
-
-        let venv = sync_environment(
+        sync_environment(
             venv,
             &resolution,
             settings.as_ref().into(),
@@ -149,10 +117,13 @@ impl CachedEnvironment {
         )
         .await?;
 
-        // Create the receipt, to indicate to future readers that the environment is complete.
-        fs_err::tokio::File::create(ok).await?;
+        // Now that the environment is complete, sync it to its content-addressed location.
+        let id = cache
+            .persist(temp_dir.into_path(), cache_entry.path())
+            .await?;
+        let root = cache.archive(&id);
 
-        Ok(Self(venv))
+        Ok(Self(PythonEnvironment::from_root(root, cache)?))
     }
 
     /// Convert the [`CachedEnvironment`] into an [`Interpreter`].

crates/uv/tests/cache_prune.rs

@@ -122,6 +122,7 @@ fn prune_cached_env() {
     DEBUG uv [VERSION] ([COMMIT] DATE)
     Pruning cache at: [CACHE_DIR]/
     DEBUG Removing dangling cache entry: [CACHE_DIR]/environments-v1/[ENTRY]
+    DEBUG Removing dangling cache entry: [CACHE_DIR]/archive-v0/[ENTRY]
     Removed [N] files ([SIZE])
     "###);
 }