mirror of
https://github.com/astral-sh/uv.git
synced 2025-10-16 21:38:31 +00:00
Support transparent Python patch version upgrades (#13954)
> NOTE: The PRs that were merged into this feature branch have all been independently reviewed. But it's also useful to see all of the changes in their final form. I've added comments to significant changes throughout the PR to aid discussion. This PR introduces transparent Python version upgrades to uv, allowing for a smoother experience when upgrading to new patch versions. Previously, upgrading Python patch versions required manual updates to each virtual environment. Now, virtual environments can transparently upgrade to newer patch versions. Due to significant changes in how uv installs and executes managed Python executables, this functionality is initially available behind a `--preview` flag. Once an installation has been made upgradeable through `--preview`, subsequent operations (like `uv venv -p 3.10` or patch upgrades) will work without requiring the flag again. This is accomplished by checking for the existence of a minor version symlink directory (or junction on Windows). ### Features * New `uv python upgrade` command to upgrade installed Python versions to the latest available patch release: ``` # Upgrade specific minor version uv python upgrade 3.12 --preview # Upgrade all installed minor versions uv python upgrade --preview ``` * Transparent upgrades also occur when installing newer patch versions: ``` uv python install 3.10.8 --preview # Automatically upgrades existing 3.10 environments uv python install 3.10.18 ``` * Support for transparently upgradeable Python `bin` installations via `--preview` flag ``` uv python install 3.13 --preview # Automatically upgrades the `bin` installation if there is a newer patch version available uv python upgrade 3.13 --preview ``` * Virtual environments can still be tied to a patch version if desired (ignoring patch upgrades): ``` uv venv -p 3.10.8 ``` ### Implementation Transparent upgrades are implemented using: * Minor version symlink directories (Unix) or junctions (Windows) * On Windows, trampolines simulate paths with junctions * Symlink directory naming follows Python build standalone format: e.g., `cpython-3.10-macos-aarch64-none` * Upgrades are scoped to the minor version key (as represented in the naming format: implementation-minor version+variant-os-arch-libc) * If the context does not provide a patch version request and the interpreter is from a managed CPython installation, the `Interpreter` used by `uv python run` will use the full symlink directory executable path when available, enabling transparently upgradeable environments created with the `venv` module (`uv run python -m venv`) New types: * `PythonMinorVersionLink`: in a sense, the core type for this PR, this is a representation of a minor version symlink directory (or junction on Windows) that points to the highest installed managed CPython patch version for a minor version key. * `PythonInstallationMinorVersionKey`: provides a view into a `PythonInstallationKey` that excludes the patch and prerelease. This is used for grouping installations by minor version key (e.g., to find the highest available patch installation for that minor version key) and for minor version directory naming. ### Compatibility * Supports virtual environments created with: * `uv venv` * `uv run python -m venv` (using managed Python that was installed or upgraded with `--preview`) * Virtual environments created within these environments * Existing virtual environments from before these changes continue to work but aren't transparently upgradeable without being recreated * Supports both standard Python (`python3.10`) and freethreaded Python (`python3.10t`) * Support for transparently upgrades is currently only available for managed CPython installations Closes #7287 Closes #7325 Closes #7892 Closes #9031 Closes #12977 --------- Co-authored-by: Zanie Blue <contact@zanie.dev>
This commit is contained in:
John Mumm
GitHub
parent
62365d4ec8
commit
e9d5780369
73 changed files with 3022 additions and 306 deletions
|
|
@ -2,6 +2,8 @@ use core::fmt;
|
|||
use std::cmp::Reverse;
|
||||
use std::ffi::OsStr;
|
||||
use std::io::{self, Write};
|
||||
#[cfg(windows)]
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
|
||||
|
|
@ -10,8 +12,11 @@ use itertools::Itertools;
|
|||
use same_file::is_same_file;
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, warn};
|
||||
use uv_configuration::PreviewMode;
|
||||
#[cfg(windows)]
|
||||
use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT;
|
||||
|
||||
use uv_fs::{LockedFile, Simplified, symlink_or_copy_file};
|
||||
use uv_fs::{LockedFile, Simplified, replace_symlink, symlink_or_copy_file};
|
||||
use uv_state::{StateBucket, StateStore};
|
||||
use uv_static::EnvVars;
|
||||
use uv_trampoline_builder::{Launcher, windows_python_launcher};
|
||||
|
|
@ -25,7 +30,9 @@ use crate::libc::LibcDetectionError;
|
|||
use crate::platform::Error as PlatformError;
|
||||
use crate::platform::{Arch, Libc, Os};
|
||||
use crate::python_version::PythonVersion;
|
||||
use crate::{PythonRequest, PythonVariant, macos_dylib, sysconfig};
|
||||
use crate::{
|
||||
PythonInstallationMinorVersionKey, PythonRequest, PythonVariant, macos_dylib, sysconfig,
|
||||
};
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
|
|
@ -51,6 +58,8 @@ pub enum Error {
|
|||
},
|
||||
#[error("Missing expected Python executable at {}", _0.user_display())]
|
||||
MissingExecutable(PathBuf),
|
||||
#[error("Missing expected target directory for Python minor version link at {}", _0.user_display())]
|
||||
MissingPythonMinorVersionLinkTargetDirectory(PathBuf),
|
||||
#[error("Failed to create canonical Python executable at {} from {}", to.user_display(), from.user_display())]
|
||||
CanonicalizeExecutable {
|
||||
from: PathBuf,
|
||||
|
|
@ -65,6 +74,13 @@ pub enum Error {
|
|||
#[source]
|
||||
err: io::Error,
|
||||
},
|
||||
#[error("Failed to create Python minor version link directory at {} from {}", to.user_display(), from.user_display())]
|
||||
PythonMinorVersionLinkDirectory {
|
||||
from: PathBuf,
|
||||
to: PathBuf,
|
||||
#[source]
|
||||
err: io::Error,
|
||||
},
|
||||
#[error("Failed to create directory for Python executable link at {}", to.user_display())]
|
||||
ExecutableDirectory {
|
||||
to: PathBuf,
|
||||
|
|
@ -339,7 +355,7 @@ impl ManagedPythonInstallation {
|
|||
|
||||
/// The path to this managed installation's Python executable.
|
||||
///
|
||||
/// If the installation has multiple execututables i.e., `python`, `python3`, etc., this will
|
||||
/// If the installation has multiple executables i.e., `python`, `python3`, etc., this will
|
||||
/// return the _canonical_ executable name which the other names link to. On Unix, this is
|
||||
/// `python{major}.{minor}{variant}` and on Windows, this is `python{exe}`.
|
||||
///
|
||||
|
|
@ -383,13 +399,11 @@ impl ManagedPythonInstallation {
|
|||
exe = std::env::consts::EXE_SUFFIX
|
||||
);
|
||||
|
||||
let executable = if cfg!(unix) || *self.implementation() == ImplementationName::GraalPy {
|
||||
self.python_dir().join("bin").join(name)
|
||||
} else if cfg!(windows) {
|
||||
self.python_dir().join(name)
|
||||
} else {
|
||||
unimplemented!("Only Windows and Unix systems are supported.")
|
||||
};
|
||||
let executable = executable_path_from_base(
|
||||
self.python_dir().as_path(),
|
||||
&name,
|
||||
&LenientImplementationName::from(*self.implementation()),
|
||||
);
|
||||
|
||||
// Workaround for python-build-standalone v20241016 which is missing the standard
|
||||
// `python.exe` executable in free-threaded distributions on Windows.
|
||||
|
|
@ -442,6 +456,10 @@ impl ManagedPythonInstallation {
|
|||
&self.key
|
||||
}
|
||||
|
||||
pub fn minor_version_key(&self) -> &PythonInstallationMinorVersionKey {
|
||||
PythonInstallationMinorVersionKey::ref_cast(&self.key)
|
||||
}
|
||||
|
||||
pub fn satisfies(&self, request: &PythonRequest) -> bool {
|
||||
match request {
|
||||
PythonRequest::File(path) => self.executable(false) == *path,
|
||||
|
|
@ -503,6 +521,30 @@ impl ManagedPythonInstallation {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure the environment contains the symlink directory (or junction on Windows)
|
||||
/// pointing to the patch directory for this minor version.
|
||||
pub fn ensure_minor_version_link(&self, preview: PreviewMode) -> Result<(), Error> {
|
||||
if let Some(minor_version_link) = PythonMinorVersionLink::from_installation(self, preview) {
|
||||
minor_version_link.create_directory()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// If the environment contains a symlink directory (or junction on Windows),
|
||||
/// update it to the latest patch directory for this minor version.
|
||||
///
|
||||
/// Unlike [`ensure_minor_version_link`], will not create a new symlink directory
|
||||
/// if one doesn't already exist,
|
||||
pub fn update_minor_version_link(&self, preview: PreviewMode) -> Result<(), Error> {
|
||||
if let Some(minor_version_link) = PythonMinorVersionLink::from_installation(self, preview) {
|
||||
if !minor_version_link.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
minor_version_link.create_directory()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure the environment is marked as externally managed with the
|
||||
/// standard `EXTERNALLY-MANAGED` file.
|
||||
pub fn ensure_externally_managed(&self) -> Result<(), Error> {
|
||||
|
|
@ -567,54 +609,8 @@ impl ManagedPythonInstallation {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a link to the managed Python executable.
|
||||
///
|
||||
/// If the file already exists at the target path, an error will be returned.
|
||||
pub fn create_bin_link(&self, target: &Path) -> Result<(), Error> {
|
||||
let python = self.executable(false);
|
||||
|
||||
let bin = target.parent().ok_or(Error::NoExecutableDirectory)?;
|
||||
fs_err::create_dir_all(bin).map_err(|err| Error::ExecutableDirectory {
|
||||
to: bin.to_path_buf(),
|
||||
err,
|
||||
})?;
|
||||
|
||||
if cfg!(unix) {
|
||||
// Note this will never copy on Unix — we use it here to allow compilation on Windows
|
||||
match symlink_or_copy_file(&python, target) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
Err(Error::MissingExecutable(python.clone()))
|
||||
}
|
||||
Err(err) => Err(Error::LinkExecutable {
|
||||
from: python,
|
||||
to: target.to_path_buf(),
|
||||
err,
|
||||
}),
|
||||
}
|
||||
} else if cfg!(windows) {
|
||||
// TODO(zanieb): Install GUI launchers as well
|
||||
let launcher = windows_python_launcher(&python, false)?;
|
||||
|
||||
// OK to use `std::fs` here, `fs_err` does not support `File::create_new` and we attach
|
||||
// error context anyway
|
||||
#[allow(clippy::disallowed_types)]
|
||||
{
|
||||
std::fs::File::create_new(target)
|
||||
.and_then(|mut file| file.write_all(launcher.as_ref()))
|
||||
.map_err(|err| Error::LinkExecutable {
|
||||
from: python,
|
||||
to: target.to_path_buf(),
|
||||
err,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
unimplemented!("Only Windows and Unix systems are supported.")
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the path is a link to this installation's binary, e.g., as created by
|
||||
/// [`ManagedPythonInstallation::create_bin_link`].
|
||||
/// [`create_bin_link`].
|
||||
pub fn is_bin_link(&self, path: &Path) -> bool {
|
||||
if cfg!(unix) {
|
||||
is_same_file(path, self.executable(false)).unwrap_or_default()
|
||||
|
|
@ -625,7 +621,11 @@ impl ManagedPythonInstallation {
|
|||
if !matches!(launcher.kind, uv_trampoline_builder::LauncherKind::Python) {
|
||||
return false;
|
||||
}
|
||||
launcher.python_path == self.executable(false)
|
||||
// We canonicalize the target path of the launcher in case it includes a minor version
|
||||
// junction directory. If canonicalization fails, we check against the launcher path
|
||||
// directly.
|
||||
dunce::canonicalize(&launcher.python_path).unwrap_or(launcher.python_path)
|
||||
== self.executable(false)
|
||||
} else {
|
||||
unreachable!("Only Windows and Unix are supported")
|
||||
}
|
||||
|
|
@ -669,6 +669,229 @@ impl ManagedPythonInstallation {
|
|||
}
|
||||
}
|
||||
|
||||
/// A representation of a minor version symlink directory (or junction on Windows)
|
||||
/// linking to the home directory of a Python installation.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PythonMinorVersionLink {
|
||||
/// The symlink directory (or junction on Windows).
|
||||
pub symlink_directory: PathBuf,
|
||||
/// The full path to the executable including the symlink directory
|
||||
/// (or junction on Windows).
|
||||
pub symlink_executable: PathBuf,
|
||||
/// The target directory for the symlink. This is the home directory for
|
||||
/// a Python installation.
|
||||
pub target_directory: PathBuf,
|
||||
}
|
||||
|
||||
impl PythonMinorVersionLink {
|
||||
/// Attempt to derive a path from an executable path that substitutes a minor
|
||||
/// version symlink directory (or junction on Windows) for the patch version
|
||||
/// directory.
|
||||
///
|
||||
/// The implementation is expected to be CPython and, on Unix, the base Python is
|
||||
/// expected to be in `<home>/bin/` on Unix. If either condition isn't true,
|
||||
/// return [`None`].
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ## Unix
|
||||
/// For a Python 3.10.8 installation in `/path/to/uv/python/cpython-3.10.8-macos-aarch64-none/bin/python3.10`,
|
||||
/// the symlink directory would be `/path/to/uv/python/cpython-3.10-macos-aarch64-none` and the executable path including the
|
||||
/// symlink directory would be `/path/to/uv/python/cpython-3.10-macos-aarch64-none/bin/python3.10`.
|
||||
///
|
||||
/// ## Windows
|
||||
/// For a Python 3.10.8 installation in `C:\path\to\uv\python\cpython-3.10.8-windows-x86_64-none\python.exe`,
|
||||
/// the junction would be `C:\path\to\uv\python\cpython-3.10-windows-x86_64-none` and the executable path including the
|
||||
/// junction would be `C:\path\to\uv\python\cpython-3.10-windows-x86_64-none\python.exe`.
|
||||
pub fn from_executable(
|
||||
executable: &Path,
|
||||
key: &PythonInstallationKey,
|
||||
preview: PreviewMode,
|
||||
) -> Option<Self> {
|
||||
let implementation = key.implementation();
|
||||
if !matches!(
|
||||
implementation,
|
||||
LenientImplementationName::Known(ImplementationName::CPython)
|
||||
) {
|
||||
// We don't currently support transparent upgrades for PyPy or GraalPy.
|
||||
return None;
|
||||
}
|
||||
let executable_name = executable
|
||||
.file_name()
|
||||
.expect("Executable file name should exist");
|
||||
let symlink_directory_name = PythonInstallationMinorVersionKey::ref_cast(key).to_string();
|
||||
let parent = executable
|
||||
.parent()
|
||||
.expect("Executable should have parent directory");
|
||||
|
||||
// The home directory of the Python installation
|
||||
let target_directory = if cfg!(unix) {
|
||||
if parent
|
||||
.components()
|
||||
.next_back()
|
||||
.is_some_and(|c| c.as_os_str() == "bin")
|
||||
{
|
||||
parent.parent()?.to_path_buf()
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
} else if cfg!(windows) {
|
||||
parent.to_path_buf()
|
||||
} else {
|
||||
unimplemented!("Only Windows and Unix systems are supported.")
|
||||
};
|
||||
let symlink_directory = target_directory.with_file_name(symlink_directory_name);
|
||||
// If this would create a circular link, return `None`.
|
||||
if target_directory == symlink_directory {
|
||||
return None;
|
||||
}
|
||||
// The full executable path including the symlink directory (or junction).
|
||||
let symlink_executable = executable_path_from_base(
|
||||
symlink_directory.as_path(),
|
||||
&executable_name.to_string_lossy(),
|
||||
implementation,
|
||||
);
|
||||
let minor_version_link = Self {
|
||||
symlink_directory,
|
||||
symlink_executable,
|
||||
target_directory,
|
||||
};
|
||||
// If preview mode is disabled, still return a `MinorVersionSymlink` for
|
||||
// existing symlinks, allowing continued operations without the `--preview`
|
||||
// flag after initial symlink directory installation.
|
||||
if preview.is_disabled() && !minor_version_link.exists() {
|
||||
return None;
|
||||
}
|
||||
Some(minor_version_link)
|
||||
}
|
||||
|
||||
pub fn from_installation(
|
||||
installation: &ManagedPythonInstallation,
|
||||
preview: PreviewMode,
|
||||
) -> Option<Self> {
|
||||
PythonMinorVersionLink::from_executable(
|
||||
installation.executable(false).as_path(),
|
||||
installation.key(),
|
||||
preview,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn create_directory(&self) -> Result<(), Error> {
|
||||
match replace_symlink(
|
||||
self.target_directory.as_path(),
|
||||
self.symlink_directory.as_path(),
|
||||
) {
|
||||
Ok(()) => {
|
||||
debug!(
|
||||
"Created link {} -> {}",
|
||||
&self.symlink_directory.user_display(),
|
||||
&self.target_directory.user_display(),
|
||||
);
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
return Err(Error::MissingPythonMinorVersionLinkTargetDirectory(
|
||||
self.target_directory.clone(),
|
||||
));
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(err) => {
|
||||
return Err(Error::PythonMinorVersionLinkDirectory {
|
||||
from: self.symlink_directory.clone(),
|
||||
to: self.target_directory.clone(),
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn exists(&self) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.symlink_directory
|
||||
.symlink_metadata()
|
||||
.map(|metadata| metadata.file_type().is_symlink())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
self.symlink_directory
|
||||
.symlink_metadata()
|
||||
.is_ok_and(|metadata| {
|
||||
// Check that this is a reparse point, which indicates this
|
||||
// is a symlink or junction.
|
||||
(metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT) != 0
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Derive the full path to an executable from the given base path and executable
|
||||
/// name. On Unix, this is, e.g., `<base>/bin/python3.10`. On Windows, this is,
|
||||
/// e.g., `<base>\python.exe`.
|
||||
fn executable_path_from_base(
|
||||
base: &Path,
|
||||
executable_name: &str,
|
||||
implementation: &LenientImplementationName,
|
||||
) -> PathBuf {
|
||||
if cfg!(unix)
|
||||
|| matches!(
|
||||
implementation,
|
||||
&LenientImplementationName::Known(ImplementationName::GraalPy)
|
||||
)
|
||||
{
|
||||
base.join("bin").join(executable_name)
|
||||
} else if cfg!(windows) {
|
||||
base.join(executable_name)
|
||||
} else {
|
||||
unimplemented!("Only Windows and Unix systems are supported.")
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a link to a managed Python executable.
|
||||
///
|
||||
/// If the file already exists at the link path, an error will be returned.
|
||||
pub fn create_link_to_executable(link: &Path, executable: PathBuf) -> Result<(), Error> {
|
||||
let link_parent = link.parent().ok_or(Error::NoExecutableDirectory)?;
|
||||
fs_err::create_dir_all(link_parent).map_err(|err| Error::ExecutableDirectory {
|
||||
to: link_parent.to_path_buf(),
|
||||
err,
|
||||
})?;
|
||||
|
||||
if cfg!(unix) {
|
||||
// Note this will never copy on Unix — we use it here to allow compilation on Windows
|
||||
match symlink_or_copy_file(&executable, link) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
Err(Error::MissingExecutable(executable.clone()))
|
||||
}
|
||||
Err(err) => Err(Error::LinkExecutable {
|
||||
from: executable,
|
||||
to: link.to_path_buf(),
|
||||
err,
|
||||
}),
|
||||
}
|
||||
} else if cfg!(windows) {
|
||||
// TODO(zanieb): Install GUI launchers as well
|
||||
let launcher = windows_python_launcher(&executable, false)?;
|
||||
|
||||
// OK to use `std::fs` here, `fs_err` does not support `File::create_new` and we attach
|
||||
// error context anyway
|
||||
#[allow(clippy::disallowed_types)]
|
||||
{
|
||||
std::fs::File::create_new(link)
|
||||
.and_then(|mut file| file.write_all(launcher.as_ref()))
|
||||
.map_err(|err| Error::LinkExecutable {
|
||||
from: executable,
|
||||
to: link.to_path_buf(),
|
||||
err,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
unimplemented!("Only Windows and Unix systems are supported.")
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(zanieb): Only used in tests now.
|
||||
/// Generate a platform portion of a key from the environment.
|
||||
pub fn platform_key_from_env() -> Result<String, Error> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue