Support transparent Python patch version upgrades (#13954)

> NOTE: The PRs that were merged into this feature branch have all been
independently reviewed. But it's also useful to see all of the changes
in their final form. I've added comments to significant changes
throughout the PR to aid discussion.

This PR introduces transparent Python version upgrades to uv, allowing
for a smoother experience when upgrading to new patch versions.
Previously, upgrading Python patch versions required manual updates to
each virtual environment. Now, virtual environments can transparently
upgrade to newer patch versions.

Due to significant changes in how uv installs and executes managed
Python executables, this functionality is initially available behind a
`--preview` flag. Once an installation has been made upgradeable through
`--preview`, subsequent operations (like `uv venv -p 3.10` or patch
upgrades) will work without requiring the flag again. This is
accomplished by checking for the existence of a minor version symlink
directory (or junction on Windows).

### Features

* New `uv python upgrade` command to upgrade installed Python versions
to the latest available patch release:
``` 
# Upgrade specific minor version 
uv python upgrade 3.12 --preview
# Upgrade all installed minor versions
uv python upgrade --preview
```
* Transparent upgrades also occur when installing newer patch versions: 
```
uv python install 3.10.8 --preview
# Automatically upgrades existing 3.10 environments
uv python install 3.10.18
```
* Support for transparently upgradeable Python `bin` installations via
`--preview` flag
```
uv python install 3.13 --preview
# Automatically upgrades the `bin` installation if there is a newer patch version available
uv python upgrade 3.13 --preview
```
* Virtual environments can still be tied to a patch version if desired
(ignoring patch upgrades):
```
uv venv -p 3.10.8
```

### Implementation

Transparent upgrades are implemented using:
* Minor version symlink directories (Unix) or junctions (Windows)
* On Windows, trampolines simulate paths with junctions
* Symlink directory naming follows Python build standalone format: e.g.,
`cpython-3.10-macos-aarch64-none`
* Upgrades are scoped to the minor version key (as represented in the
naming format: implementation-minor version+variant-os-arch-libc)
* If the context does not provide a patch version request and the
interpreter is from a managed CPython installation, the `Interpreter`
used by `uv python run` will use the full symlink directory executable
path when available, enabling transparently upgradeable environments
created with the `venv` module (`uv run python -m venv`)

New types:
* `PythonMinorVersionLink`: in a sense, the core type for this PR, this
is a representation of a minor version symlink directory (or junction on
Windows) that points to the highest installed managed CPython patch
version for a minor version key.
* `PythonInstallationMinorVersionKey`: provides a view into a
`PythonInstallationKey` that excludes the patch and prerelease. This is
used for grouping installations by minor version key (e.g., to find the
highest available patch installation for that minor version key) and for
minor version directory naming.

### Compatibility

* Supports virtual environments created with:
  * `uv venv`
* `uv run python -m venv` (using managed Python that was installed or
upgraded with `--preview`)
  * Virtual environments created within these environments
* Existing virtual environments from before these changes continue to
work but aren't transparently upgradeable without being recreated
* Supports both standard Python (`python3.10`) and freethreaded Python
(`python3.10t`)
* Support for transparently upgrades is currently only available for
managed CPython installations

Closes #7287
Closes #7325
Closes #7892
Closes #9031
Closes #12977

---------

Co-authored-by: Zanie Blue <contact@zanie.dev>

crates/uv-virtualenv/Cargo.toml

@@ -20,6 +20,7 @@ doctest = false
 workspace = true
 
 [dependencies]
+uv-configuration = { workspace = true }
 uv-fs = { workspace = true }
 uv-pypi-types = { workspace = true }
 uv-python = { workspace = true }

crates/uv-virtualenv/src/lib.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 
 use thiserror::Error;
 
+use uv_configuration::PreviewMode;
 use uv_python::{Interpreter, PythonEnvironment};
 
 mod virtualenv;
@@ -15,6 +16,8 @@ pub enum Error {
         "Could not find a suitable Python executable for the virtual environment based on the interpreter: {0}"
     )]
     NotFound(String),
+    #[error(transparent)]
+    Python(#[from] uv_python::managed::Error),
 }
 
 /// The value to use for the shell prompt when inside a virtual environment.
@@ -50,6 +53,8 @@ pub fn create_venv(
     allow_existing: bool,
     relocatable: bool,
     seed: bool,
+    upgradeable: bool,
+    preview: PreviewMode,
 ) -> Result<PythonEnvironment, Error> {
     // Create the virtualenv at the given location.
     let virtualenv = virtualenv::create(
@@ -60,6 +65,8 @@ pub fn create_venv(
         allow_existing,
         relocatable,
         seed,
+        upgradeable,
+        preview,
     )?;
 
     // Create the corresponding `PythonEnvironment`.

crates/uv-virtualenv/src/virtualenv.rs

@@ -10,8 +10,10 @@ use fs_err::File;
 use itertools::Itertools;
 use tracing::debug;
 
+use uv_configuration::PreviewMode;
 use uv_fs::{CWD, Simplified, cachedir};
 use uv_pypi_types::Scheme;
+use uv_python::managed::{PythonMinorVersionLink, create_link_to_executable};
 use uv_python::{Interpreter, VirtualEnvironment};
 use uv_shell::escape_posix_for_single_quotes;
 use uv_version::version;
@@ -53,6 +55,8 @@ pub(crate) fn create(
     allow_existing: bool,
     relocatable: bool,
     seed: bool,
+    upgradeable: bool,
+    preview: PreviewMode,
 ) -> Result<VirtualEnvironment, Error> {
     // Determine the base Python executable; that is, the Python executable that should be
     // considered the "base" for the virtual environment.
@@ -143,13 +147,49 @@ pub(crate) fn create(
     // Create a `.gitignore` file to ignore all files in the venv.
     fs::write(location.join(".gitignore"), "*")?;
 
+    let executable_target = if upgradeable && interpreter.is_standalone() {
+        if let Some(minor_version_link) = PythonMinorVersionLink::from_executable(
+            base_python.as_path(),
+            &interpreter.key(),
+            preview,
+        ) {
+            if !minor_version_link.exists() {
+                base_python.clone()
+            } else {
+                let debug_symlink_term = if cfg!(windows) {
+                    "junction"
+                } else {
+                    "symlink directory"
+                };
+                debug!(
+                    "Using {} {} instead of base Python path: {}",
+                    debug_symlink_term,
+                    &minor_version_link.symlink_directory.display(),
+                    &base_python.display()
+                );
+                minor_version_link.symlink_executable.clone()
+            }
+        } else {
+            base_python.clone()
+        }
+    } else {
+        base_python.clone()
+    };
+
     // Per PEP 405, the Python `home` is the parent directory of the interpreter.
-    let python_home = base_python.parent().ok_or_else(|| {
-        io::Error::new(
-            io::ErrorKind::NotFound,
-            "The Python interpreter needs to have a parent directory",
-        )
-    })?;
+    // In preview mode, for standalone interpreters, this `home` value will include a
+    // symlink directory on Unix or junction on Windows to enable transparent Python patch
+    // upgrades.
+    let python_home = executable_target
+        .parent()
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::NotFound,
+                "The Python interpreter needs to have a parent directory",
+            )
+        })?
+        .to_path_buf();
+    let python_home = python_home.as_path();
 
     // Different names for the python interpreter
     fs::create_dir_all(&scripts)?;
@@ -157,7 +197,7 @@ pub(crate) fn create(
 
     #[cfg(unix)]
     {
-        uv_fs::replace_symlink(&base_python, &executable)?;
+        uv_fs::replace_symlink(&executable_target, &executable)?;
         uv_fs::replace_symlink(
             "python",
             scripts.join(format!("python{}", interpreter.python_major())),
@@ -184,91 +224,102 @@ pub(crate) fn create(
         }
     }
 
-    // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
+    // On Windows, we use trampolines that point to an executable target. For standalone
+    // interpreters, this target path includes a minor version junction to enable
+    // transparent upgrades.
     if cfg!(windows) {
-        copy_launcher_windows(
-            WindowsExecutable::Python,
-            interpreter,
-            &base_python,
-            &scripts,
-            python_home,
-        )?;
-
-        if interpreter.markers().implementation_name() == "graalpy" {
-            copy_launcher_windows(
-                WindowsExecutable::GraalPy,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PythonMajor,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
+        if interpreter.is_standalone() {
+            let target = scripts.join(WindowsExecutable::Python.exe(interpreter));
+            create_link_to_executable(target.as_path(), executable_target.clone())
+                .map_err(Error::Python)?;
+            let targetw = scripts.join(WindowsExecutable::Pythonw.exe(interpreter));
+            create_link_to_executable(targetw.as_path(), executable_target)
+                .map_err(Error::Python)?;
         } else {
             copy_launcher_windows(
-                WindowsExecutable::Pythonw,
+                WindowsExecutable::Python,
                 interpreter,
                 &base_python,
                 &scripts,
                 python_home,
             )?;
-        }
 
-        if interpreter.markers().implementation_name() == "pypy" {
-            copy_launcher_windows(
-                WindowsExecutable::PythonMajor,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PythonMajorMinor,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PyPy,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PyPyMajor,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PyPyMajorMinor,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PyPyw,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
-            copy_launcher_windows(
-                WindowsExecutable::PyPyMajorMinorw,
-                interpreter,
-                &base_python,
-                &scripts,
-                python_home,
-            )?;
+            if interpreter.markers().implementation_name() == "graalpy" {
+                copy_launcher_windows(
+                    WindowsExecutable::GraalPy,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PythonMajor,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+            } else {
+                copy_launcher_windows(
+                    WindowsExecutable::Pythonw,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+            }
+
+            if interpreter.markers().implementation_name() == "pypy" {
+                copy_launcher_windows(
+                    WindowsExecutable::PythonMajor,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PythonMajorMinor,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PyPy,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PyPyMajor,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PyPyMajorMinor,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PyPyw,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+                copy_launcher_windows(
+                    WindowsExecutable::PyPyMajorMinorw,
+                    interpreter,
+                    &base_python,
+                    &scripts,
+                    python_home,
+                )?;
+            }
         }
     }