mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 21:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge 0fd8580e20 into f609e1ddaf

Browse source
This commit is contained in:
Charlie Marsh 2025-07-05 01:04:21 +03:30 • committed by GitHub
commit 49eecf9a28
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 159 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/uv-python/src/environment.rs
View file

@ -174,7 +174,7 @@ impl PythonEnvironment {
/// N.B. This function also works for system Python environments and users depend on this.
pub fn from_root(root: impl AsRef<Path>, cache: &Cache) -> Result<Self, Error> {
debug!(
"Checking for Python environment at `{}`",
"Checking for Python environment at: `{}`",
root.as_ref().user_display()
);
match root.as_ref().try_exists() {

166
crates/uv/src/commands/project/environment.rs
View file

@ -17,6 +17,69 @@ use crate::commands::project::{
use crate::printer::Printer;
use crate::settings::{NetworkSettings, ResolverInstallerSettings};
/// An ephemeral [`PythonEnvironment`] for running an individual command.
#[derive(Debug)]
pub(crate) struct EphemeralEnvironment(PythonEnvironment);
impl From<PythonEnvironment> for EphemeralEnvironment {
fn from(environment: PythonEnvironment) -> Self {
Self(environment)
}
}
impl From<EphemeralEnvironment> for PythonEnvironment {
fn from(environment: EphemeralEnvironment) -> Self {
environment.0
}
}
impl EphemeralEnvironment {
/// Set the ephemeral overlay for a Python environment.
#[allow(clippy::result_large_err)]
pub(crate) fn set_overlay(&self, contents: impl AsRef<[u8]>) -> Result<(), ProjectError> {
let site_packages = self
.0
.site_packages()
.next()
.ok_or(ProjectError::NoSitePackages)?;
let overlay_path = site_packages.join("_uv_ephemeral_overlay.pth");
fs_err::write(overlay_path, contents)?;
Ok(())
}
/// Enable system site packages for a Python environment.
#[allow(clippy::result_large_err)]
pub(crate) fn set_system_site_packages(&self) -> Result<(), ProjectError> {
self.0
.set_pyvenv_cfg("include-system-site-packages", "true")?;
Ok(())
}
/// Set the `extends-environment` key in the `pyvenv.cfg` file to the given path.
///
/// Ephemeral environments created by `uv run --with` extend a parent (virtual or system)
/// environment by adding a `.pth` file to the ephemeral environment's `site-packages`
/// directory. The `pth` file contains Python code to dynamically add the parent
/// environment's `site-packages` directory to Python's import search paths in addition to
/// the ephemeral environment's `site-packages` directory. This works well at runtime, but
/// is too dynamic for static analysis tools like ty to understand. As such, we
/// additionally write the `sys.prefix` of the parent environment to to the
/// `extends-environment` key of the ephemeral environment's `pyvenv.cfg` file, making it
/// easier for these tools to statically and reliably understand the relationship between
/// the two environments.
#[allow(clippy::result_large_err)]
pub(crate) fn set_parent_environment(
&self,
parent_environment_sys_prefix: &Path,
) -> Result<(), ProjectError> {
self.0.set_pyvenv_cfg(
"extends-environment",
&parent_environment_sys_prefix.escape_for_python(),
)?;
Ok(())
}
}
/// A [`PythonEnvironment`] stored in the cache.
#[derive(Debug)]
pub(crate) struct CachedEnvironment(PythonEnvironment);
@ -44,15 +107,13 @@ impl CachedEnvironment {
printer: Printer,
preview: PreviewMode,
) -> Result<Self, ProjectError> {
// Resolve the "base" interpreter, which resolves to an underlying parent interpreter if the
// given interpreter is a virtual environment.
let base_interpreter = Self::base_interpreter(interpreter, cache)?;
let interpreter = Self::base_interpreter(interpreter, cache)?;
// Resolve the requirements with the interpreter.
let resolution = Resolution::from(
resolve_environment(
spec,
&base_interpreter,
&interpreter,
build_constraints.clone(),
&settings.resolver,
network_settings,
@ -80,29 +141,20 @@ impl CachedEnvironment {
// Use the canonicalized base interpreter path since that's the interpreter we performed the
// resolution with and the interpreter the environment will be created with.
//
// We also include the canonicalized `sys.prefix` of the non-base interpreter, that is, the
// virtual environment's path. Originally, we shared cached environments independent of the
// environment they'd be layered on top of. However, this causes collisions as the overlay
// `.pth` file can be overridden by another instance of uv. Including this element in the key
// avoids this problem at the cost of creating separate cached environments for identical
// `--with` invocations across projects. We use `sys.prefix` rather than `sys.executable` so
// we can canonicalize it without invalidating the purpose of the element — it'd probably be
// safe to just use the absolute `sys.executable` as well.
//
// TODO(zanieb): Since we're not sharing these environmments across projects, we should move
// [`CachedEvnvironment::set_overlay`] etc. here since the values there should be constant
// now.
// We cache environments independent of the environment they'd be layered on top of. The
// assumption is such that the environment will _not_ be modified by the user or uv;
// otherwise, we risk cache poisoning. For example, if we were to write a `.pth` file to
// the cached environment, it would be shared across all projects that use the same
// interpreter and the same cached dependencies.
//
// TODO(zanieb): We should include the version of the base interpreter in the hash, so if
// the interpreter at the canonicalized path changes versions we construct a new
// environment.
let environment_hash = cache_digest(&(
&canonicalize_executable(base_interpreter.sys_executable())?,
&interpreter.sys_prefix().canonicalize()?,
));
let interpreter_hash =
cache_digest(&canonicalize_executable(interpreter.sys_executable())?);
// Search in the content-addressed cache.
let cache_entry = cache.entry(CacheBucket::Environments, environment_hash, resolution_hash);
let cache_entry = cache.entry(CacheBucket::Environments, interpreter_hash, resolution_hash);
if cache.refresh().is_none() {
if let Ok(root) = cache.resolve_link(cache_entry.path()) {
@ -116,7 +168,7 @@ impl CachedEnvironment {
let temp_dir = cache.venv_dir()?;
let venv = uv_virtualenv::create_venv(
temp_dir.path(),
base_interpreter,
interpreter,
uv_virtualenv::Prompt::None,
false,
false,
@ -150,76 +202,6 @@ impl CachedEnvironment {
Ok(Self(PythonEnvironment::from_root(root, cache)?))
}
/// Set the ephemeral overlay for a Python environment.
#[allow(clippy::result_large_err)]
pub(crate) fn set_overlay(&self, contents: impl AsRef<[u8]>) -> Result<(), ProjectError> {
let site_packages = self
.0
.site_packages()
.next()
.ok_or(ProjectError::NoSitePackages)?;
let overlay_path = site_packages.join("_uv_ephemeral_overlay.pth");
fs_err::write(overlay_path, contents)?;
Ok(())
}
/// Clear the ephemeral overlay for a Python environment, if it exists.
#[allow(clippy::result_large_err)]
pub(crate) fn clear_overlay(&self) -> Result<(), ProjectError> {
let site_packages = self
.0
.site_packages()
.next()
.ok_or(ProjectError::NoSitePackages)?;
let overlay_path = site_packages.join("_uv_ephemeral_overlay.pth");
match fs_err::remove_file(overlay_path) {
Ok(()) => (),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => (),
Err(err) => return Err(ProjectError::OverlayRemoval(err)),
}
Ok(())
}
/// Enable system site packages for a Python environment.
#[allow(clippy::result_large_err)]
pub(crate) fn set_system_site_packages(&self) -> Result<(), ProjectError> {
self.0
.set_pyvenv_cfg("include-system-site-packages", "true")?;
Ok(())
}
/// Disable system site packages for a Python environment.
#[allow(clippy::result_large_err)]
pub(crate) fn clear_system_site_packages(&self) -> Result<(), ProjectError> {
self.0
.set_pyvenv_cfg("include-system-site-packages", "false")?;
Ok(())
}
/// Set the `extends-environment` key in the `pyvenv.cfg` file to the given path.
///
/// Ephemeral environments created by `uv run --with` extend a parent (virtual or system)
/// environment by adding a `.pth` file to the ephemeral environment's `site-packages`
/// directory. The `pth` file contains Python code to dynamically add the parent
/// environment's `site-packages` directory to Python's import search paths in addition to
/// the ephemeral environment's `site-packages` directory. This works well at runtime, but
/// is too dynamic for static analysis tools like ty to understand. As such, we
/// additionally write the `sys.prefix` of the parent environment to to the
/// `extends-environment` key of the ephemeral environment's `pyvenv.cfg` file, making it
/// easier for these tools to statically and reliably understand the relationship between
/// the two environments.
#[allow(clippy::result_large_err)]
pub(crate) fn set_parent_environment(
&self,
parent_environment_sys_prefix: &Path,
) -> Result<(), ProjectError> {
self.0.set_pyvenv_cfg(
"extends-environment",
&parent_environment_sys_prefix.escape_for_python(),
)?;
Ok(())
}
/// Return the [`Interpreter`] to use for the cached environment, based on a given
/// [`Interpreter`].
///

3
crates/uv/src/commands/project/mod.rs
View file

@ -200,9 +200,6 @@ pub(crate) enum ProjectError {
#[error("Failed to parse PEP 723 script metadata")]
Pep723ScriptTomlParse(#[source] toml::de::Error),
#[error("Failed to remove ephemeral overlay")]
OverlayRemoval(#[source] std::io::Error),
#[error("Failed to find `site-packages` directory for environment")]
NoSitePackages,

76
crates/uv/src/commands/project/run.rs
View file

@ -44,7 +44,7 @@ use crate::commands::pip::loggers::{
DefaultInstallLogger, DefaultResolveLogger, SummaryInstallLogger, SummaryResolveLogger,
};
use crate::commands::pip::operations::Modifications;
use crate::commands::project::environment::CachedEnvironment;
use crate::commands::project::environment::{CachedEnvironment, EphemeralEnvironment};
use crate::commands::project::install_target::InstallTarget;
use crate::commands::project::lock::LockMode;
use crate::commands::project::lock_target::LockTarget;
@ -939,7 +939,7 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
// If necessary, create an environment for the ephemeral requirements or command.
let base_site_packages = SitePackages::from_interpreter(&base_interpreter)?;
let ephemeral_env = match spec {
let requirements_env = match spec {
None => None,
Some(spec)
if can_skip_ephemeral(&spec, &base_interpreter, &base_site_packages, &settings) =>
@ -947,7 +947,7 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
None
}
Some(spec) => {
debug!("Syncing ephemeral requirements");
debug!("Syncing `--with` requirements to cached environment");
// Read the build constraints from the lock file.
let build_constraints = base_lock
@ -1008,24 +1008,62 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
Err(err) => return Err(err.into()),
};
Some(environment)
Some(PythonEnvironment::from(environment))
}
};
// If we're running in an ephemeral environment, add a path file to enable loading of
// the base environment's site packages. Setting `PYTHONPATH` is insufficient, as it doesn't
// resolve `.pth` files in the base environment.
// If we're layering requirements atop the project environment, run the command in an ephemeral,
// isolated environment. Otherwise, modifications to the "active virtual environment" would
// poison the cache.
let ephemeral_dir = requirements_env
.as_ref()
.map(|_| cache.venv_dir())
.transpose()?;
let ephemeral_env = ephemeral_dir
.as_ref()
.map(|dir| {
debug!(
"Creating ephemeral environment at: `{}`",
dir.path().simplified_display()
);
uv_virtualenv::create_venv(
dir.path(),
base_interpreter.clone(),
uv_virtualenv::Prompt::None,
false,
false,
false,
false,
false,
preview,
)
})
.transpose()?
.map(EphemeralEnvironment::from);
// If we're running in an ephemeral environment, add a path file to enable loading from the
// `--with` requirements environment and the project environment site packages.
//
// `sitecustomize.py` would be an alternative, but it can be shadowed by an existing such
// module in the python installation.
// Setting `PYTHONPATH` is insufficient, as it doesn't resolve `.pth` files in the base
// environment. Adding `sitecustomize.py` would be an alternative, but it can be shadowed by an
// existing such module in the python installation.
if let Some(ephemeral_env) = ephemeral_env.as_ref() {
let site_packages = base_interpreter
if let Some(requirements_env) = requirements_env.as_ref() {
let requirements_site_packages =
requirements_env.site_packages().next().ok_or_else(|| {
anyhow!("Requirements environment has no site packages directory")
})?;
let base_site_packages = base_interpreter
.site_packages()
.next()
.ok_or_else(|| ProjectError::NoSitePackages)?;
.ok_or_else(|| anyhow!("Base environment has no site packages directory"))?;
ephemeral_env.set_overlay(format!(
"import site; site.addsitedir(\"{}\")",
site_packages.escape_for_python()
"import site; site.addsitedir(\"{}\"); site.addsitedir(\"{}\");",
base_site_packages.escape_for_python(),
requirements_site_packages.escape_for_python(),
))?;
// Write the `sys.prefix` of the parent environment to the `extends-environment` key of the `pyvenv.cfg`
@ -1045,17 +1083,17 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
.is_ok_and(|cfg| cfg.include_system_site_packages())
{
ephemeral_env.set_system_site_packages()?;
} else {
ephemeral_env.clear_system_site_packages()?;
}
}
}
// Cast from `CachedEnvironment` to `PythonEnvironment`.
// Cast to `PythonEnvironment`.
let ephemeral_env = ephemeral_env.map(PythonEnvironment::from);
// Determine the Python interpreter to use for the command, if necessary.
let interpreter = ephemeral_env
.as_ref()
.or(requirements_env.as_ref())
.map_or_else(|| &base_interpreter, |env| env.interpreter());
// Check if any run command is given.
@ -1138,6 +1176,12 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
.as_ref()
.map(PythonEnvironment::scripts)
.into_iter()
.chain(
requirements_env
.as_ref()
.map(PythonEnvironment::scripts)
.into_iter(),
)
.chain(std::iter::once(base_interpreter.scripts()))
.chain(
// On Windows, non-virtual Python distributions put `python.exe` in the top-level

4
crates/uv/src/commands/tool/run.rs
View file

@ -1080,9 +1080,5 @@ async fn get_or_create_environment(
},
};
// Clear any existing overlay.
environment.clear_overlay()?;
environment.clear_system_site_packages()?;
Ok((from, environment.into()))
}

2
crates/uv/tests/it/lock.rs
View file

@ -15874,7 +15874,7 @@ fn lock_explicit_default_index() -> Result<()> {
DEBUG Adding root workspace member: `[TEMP_DIR]/`
DEBUG No Python version file found in workspace: [TEMP_DIR]/
DEBUG Using Python request `>=3.12` from `requires-python` metadata
DEBUG Checking for Python environment at `.venv`
DEBUG Checking for Python environment at: `.venv`
DEBUG The project environment's Python version satisfies the request: `Python >=3.12`
DEBUG Using request timeout of [TIME]
DEBUG Found static `pyproject.toml` for: project @ file://[TEMP_DIR]/

13
crates/uv/tests/it/run.rs
View file

@ -1302,7 +1302,6 @@ fn run_with_pyvenv_cfg_file() -> Result<()> {
uv = [UV_VERSION]
version_info = 3.12.[X]
include-system-site-packages = false
relocatable = true
extends-environment = [PARENT_VENV]
@ -4778,7 +4777,6 @@ fn run_groups_include_requires_python() -> Result<()> {
baz = ["iniconfig"]
dev = ["sniffio", {include-group = "foo"}, {include-group = "baz"}]
[tool.uv.dependency-groups]
foo = {requires-python="<3.13"}
bar = {requires-python=">=3.13"}
@ -4923,8 +4921,8 @@ fn run_repeated() -> Result<()> {
Resolved 1 package in [TIME]
"###);
// Re-running as a tool does require reinstalling `typing-extensions`, since the base venv is
// different.
// Re-running as a tool doesn't require reinstalling `typing-extensions`, since the environment
// is cached.
uv_snapshot!(
context.filters(),
context.tool_run().arg("--with").arg("typing-extensions").arg("python").arg("-c").arg("import typing_extensions; import iniconfig"), @r#"
@ -4934,8 +4932,6 @@ fn run_repeated() -> Result<()> {
----- stderr -----
Resolved 1 package in [TIME]
Installed 1 package in [TIME]
+ typing-extensions==4.10.0
Traceback (most recent call last):
File "<string>", line 1, in <module>
import typing_extensions; import iniconfig
@ -4982,8 +4978,7 @@ fn run_without_overlay() -> Result<()> {
+ typing-extensions==4.10.0
"###);
// Import `iniconfig` in the context of a `tool run` command, which should fail. Note that
// typing-extensions gets installed again, because the venv is not shared.
// Import `iniconfig` in the context of a `tool run` command, which should fail.
uv_snapshot!(
context.filters(),
context.tool_run().arg("--with").arg("typing-extensions").arg("python").arg("-c").arg("import typing_extensions; import iniconfig"), @r#"
@ -4993,8 +4988,6 @@ fn run_without_overlay() -> Result<()> {
----- stderr -----
Resolved 1 package in [TIME]
Installed 1 package in [TIME]
+ typing-extensions==4.10.0
Traceback (most recent call last):
File "<string>", line 1, in <module>
import typing_extensions; import iniconfig