mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 21:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Query interpreter to determine correct virtualenv paths (#2188)

Browse source 
## Summary

This PR migrates our virtualenv creation from a setup that assumes prior
knowledge of the correct paths, to a technique borrowed from
`virtualenv` whereby we use `sysconfig` and `distutils` to determine the
paths. The general trick is to grab the expected paths with `sysconfig`,
then make them all relative, then make them absolute for a given
directory.

Closes #2095.
Closes #2153.
This commit is contained in:
Charlie Marsh 2024-03-05 13:13:24 -08:00 • committed by GitHub
parent a8ac7b1eb4
commit 0f6fc117c1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 171 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -4662,6 +4662,7 @@ dependencies = [
"clap",
"directories",
"fs-err",
"pathdiff",
"platform-host",
"pypi-types",
"serde",

1
crates/pypi-types/src/scheme.rs
View file

@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize};
/// See: <https://docs.python.org/3.12/library/sysconfig.html#installation-paths>
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Scheme {
pub stdlib: PathBuf,
pub purelib: PathBuf,
pub platlib: PathBuf,
pub scripts: PathBuf,

124
crates/uv-interpreter/src/get_interpreter_info.py
View file

@ -7,7 +7,6 @@ Exit Codes:
3: Python version 3 or newer is required
"""
import json
import os
import platform
@ -67,9 +66,126 @@ python_full_version = platform.python_version()
# [3]: https://github.com/pypa/packaging/issues/678#issuecomment-1436033646
# [4]: https://github.com/astral-sh/uv/issues/1357#issuecomment-1947645243
# [5]: https://github.com/pypa/packaging/blob/085ff41692b687ae5b0772a55615b69a5b677be9/packaging/version.py#L168-L193
if len(python_full_version) > 0 and python_full_version[-1] == '+':
if len(python_full_version) > 0 and python_full_version[-1] == "+":
python_full_version = python_full_version[:-1]
def get_virtualenv():
"""Return the expected Scheme for virtualenvs created by this interpreter.
The paths returned should be relative to a root directory.
This is based on virtualenv's path discovery logic:
https://github.com/pypa/virtualenv/blob/5cd543fdf8047600ff2737babec4a635ad74d169/src/virtualenv/discovery/py_info.py#L80C9-L80C17
"""
scheme_names = sysconfig.get_scheme_names()
# Determine the scheme to use, if any.
if "venv" in scheme_names:
sysconfig_scheme = "venv"
elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names:
# debian / ubuntu python 3.10 without `python3-distutils` will report
# mangled `local/bin` / etc. names for the default prefix
# intentionally select `posix_prefix` which is the unaltered posix-like paths
sysconfig_scheme = "posix_prefix"
else:
sysconfig_scheme = None
# Use `sysconfig`, if available.
if sysconfig_scheme:
import re
sysconfig_paths = {
i: sysconfig.get_path(i, expand=False, scheme=sysconfig_scheme)
for i in sysconfig.get_path_names()
}
# Determine very configuration variable that we need to resolve.
config_var_keys = set()
conf_var_re = re.compile(r"\{\w+}")
for element in sysconfig_paths.values():
for k in conf_var_re.findall(element):
config_var_keys.add(k[1:-1])
config_var_keys.add("PYTHONFRAMEWORK")
# Look them up.
sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys}
# Information about the prefix (determines the Python home).
prefix = os.path.abspath(sys.prefix)
base_prefix = os.path.abspath(sys.base_prefix)
# Information about the exec prefix (dynamic stdlib modules).
base_exec_prefix = os.path.abspath(sys.base_exec_prefix)
exec_prefix = os.path.abspath(sys.exec_prefix)
# Set any prefixes to empty, which makes the resulting paths relative.
prefixes = prefix, exec_prefix, base_prefix, base_exec_prefix
sysconfig_vars.update(
{k: "" if v in prefixes else v for k, v in sysconfig_vars.items()}
)
def expand_path(path: str) -> str:
return path.format(**sysconfig_vars).replace("/", os.sep).lstrip(os.sep)
return {
"purelib": expand_path(sysconfig_paths["purelib"]),
"platlib": expand_path(sysconfig_paths["platlib"]),
"include": expand_path(sysconfig_paths["include"]),
"scripts": expand_path(sysconfig_paths["scripts"]),
"data": expand_path(sysconfig_paths["data"]),
}
else:
# Use distutils primarily because that's what pip does.
# https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L249
import warnings
with warnings.catch_warnings(): # disable warning for PEP-632
warnings.simplefilter("ignore")
from distutils import dist
from distutils.command.install import SCHEME_KEYS
d = dist.Distribution({"script_args": "--no-user-cfg"})
if hasattr(sys, "_framework"):
sys._framework = None
with warnings.catch_warnings():
warnings.simplefilter("ignore")
i = d.get_command_obj("install", create=True)
i.prefix = os.sep
i.finalize_options()
distutils_paths = {
key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep)
for key in SCHEME_KEYS
}
return {
"purelib": distutils_paths["purelib"],
"platlib": distutils_paths["platlib"],
"include": os.path.dirname(distutils_paths["headers"]),
"scripts": distutils_paths["scripts"],
"data": distutils_paths["data"],
}
def get_scheme():
"""Return the Scheme for the current interpreter.
The paths returned should be absolute.
"""
# TODO(charlie): Use distutils on required Python distributions.
paths = sysconfig.get_paths()
return {
"purelib": paths["purelib"],
"platlib": paths["platlib"],
"include": paths["include"],
"scripts": paths["scripts"],
"data": paths["data"],
}
markers = {
"implementation_name": implementation_name,
"implementation_version": implementation_version,
@ -90,6 +206,8 @@ interpreter_info = {
"prefix": sys.prefix,
"base_executable": getattr(sys, "_base_executable", None),
"sys_executable": sys.executable,
"scheme": sysconfig.get_paths(),
"stdlib": sysconfig.get_path("stdlib"),
"scheme": get_scheme(),
"virtualenv": get_virtualenv(),
}
print(json.dumps(interpreter_info))

55
crates/uv-interpreter/src/interpreter.rs
View file

@ -28,11 +28,13 @@ pub struct Interpreter {
platform: Platform,
markers: Box<MarkerEnvironment>,
scheme: Scheme,
virtualenv: Scheme,
prefix: PathBuf,
base_exec_prefix: PathBuf,
base_prefix: PathBuf,
base_executable: Option<PathBuf>,
sys_executable: PathBuf,
stdlib: PathBuf,
tags: OnceCell<Tags>,
}
@ -51,11 +53,13 @@ impl Interpreter {
platform,
markers: Box::new(info.markers),
scheme: info.scheme,
virtualenv: info.virtualenv,
prefix: info.prefix,
base_exec_prefix: info.base_exec_prefix,
base_prefix: info.base_prefix,
base_executable: info.base_executable,
sys_executable: info.sys_executable,
stdlib: info.stdlib,
tags: OnceCell::new(),
})
}
@ -66,7 +70,13 @@ impl Interpreter {
platform,
markers: Box::new(markers),
scheme: Scheme {
stdlib: PathBuf::from("/dev/null"),
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
scripts: PathBuf::from("/dev/null"),
data: PathBuf::from("/dev/null"),
},
virtualenv: Scheme {
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
@ -78,6 +88,7 @@ impl Interpreter {
base_prefix: PathBuf::from("/dev/null"),
base_executable: None,
sys_executable: PathBuf::from("/dev/null"),
stdlib: PathBuf::from("/dev/null"),
tags: OnceCell::new(),
}
}
@ -240,7 +251,7 @@ impl Interpreter {
return None;
}
let Ok(contents) = fs::read_to_string(self.scheme.stdlib.join("EXTERNALLY-MANAGED")) else {
let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
return None;
};
@ -345,6 +356,11 @@ impl Interpreter {
&self.sys_executable
}
/// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn stdlib(&self) -> &Path {
&self.stdlib
}
/// Return the `purelib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn purelib(&self) -> &Path {
&self.scheme.purelib
@ -370,21 +386,9 @@ impl Interpreter {
&self.scheme.include
}
/// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn stdlib(&self) -> &Path {
&self.scheme.stdlib
}
/// Return the name of the Python directory used to build the path to the
/// `site-packages` directory.
///
/// If one could not be determined, then `python` is returned.
pub fn site_packages_python(&self) -> &str {
if self.implementation_name() == "pypy" {
"pypy"
} else {
"python"
}
/// Return the [`Scheme`] for a virtual environment created by this [`Interpreter`].
pub fn virtualenv(&self) -> &Scheme {
&self.virtualenv
}
/// Return the [`Layout`] environment used to install wheels into this interpreter.
@ -394,7 +398,6 @@ impl Interpreter {
sys_executable: self.sys_executable().to_path_buf(),
os_name: self.markers.os_name.clone(),
scheme: Scheme {
stdlib: self.stdlib().to_path_buf(),
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
@ -403,8 +406,7 @@ impl Interpreter {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"{}{}.{}",
self.site_packages_python(),
"python{}.{}",
self.python_major(),
self.python_minor()
))
@ -435,11 +437,13 @@ impl ExternallyManaged {
struct InterpreterInfo {
markers: MarkerEnvironment,
scheme: Scheme,
virtualenv: Scheme,
prefix: PathBuf,
base_exec_prefix: PathBuf,
base_prefix: PathBuf,
base_executable: Option<PathBuf>,
sys_executable: PathBuf,
stdlib: PathBuf,
}
impl InterpreterInfo {
@ -635,13 +639,20 @@ mod tests {
"base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
"prefix": "/home/ferris/projects/uv/.venv",
"sys_executable": "/home/ferris/projects/uv/.venv/bin/python",
"stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12",
"scheme": {
"data": "/home/ferris/.pyenv/versions/3.12.0",
"include": "/home/ferris/.pyenv/versions/3.12.0/include",
"platlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"purelib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"scripts": "/home/ferris/.pyenv/versions/3.12.0/bin",
"stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12"
"scripts": "/home/ferris/.pyenv/versions/3.12.0/bin"
},
"virtualenv": {
"data": "",
"include": "include",
"platlib": "lib/python3.12/site-packages",
"purelib": "lib/python3.12/site-packages",
"scripts": "bin"
}
}
"##};

1
crates/uv-virtualenv/Cargo.toml
View file

@ -32,6 +32,7 @@ cachedir = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
directories = { workspace = true }
fs-err = { workspace = true }
pathdiff = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }

55
crates/uv-virtualenv/src/bare.rs
View file

@ -8,9 +8,9 @@ use std::path::Path;
use fs_err as fs;
use fs_err::File;
use pypi_types::Scheme;
use tracing::info;
use pypi_types::Scheme;
use uv_fs::Simplified;
use uv_interpreter::{Interpreter, Virtualenv};
@ -124,7 +124,7 @@ pub fn create_bare_venv(
} else {
unimplemented!("Only Windows and Unix are supported")
};
let scripts = location.join(bin_name);
let scripts = location.join(&interpreter.virtualenv().scripts);
let prompt = match prompt {
Prompt::CurrentDirectoryName => env::current_dir()?
.file_name()
@ -143,7 +143,6 @@ pub fn create_bare_venv(
fs::create_dir(&scripts)?;
let executable = scripts.join(format!("python{EXE_SUFFIX}"));
// No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
#[cfg(unix)]
{
use fs_err::os::unix::fs::symlink;
@ -163,6 +162,7 @@ pub fn create_bare_venv(
)?;
}
// No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
#[cfg(windows)]
{
// https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267
@ -205,18 +205,11 @@ pub fn create_bare_venv(
// Add all the activate scripts for different shells
for (name, template) in ACTIVATE_TEMPLATES {
let relative_site_packages = if cfg!(unix) {
format!(
"../lib/{}{}.{}/site-packages",
interpreter.site_packages_python(),
interpreter.python_major(),
interpreter.python_minor(),
)
} else if cfg!(windows) {
"../Lib/site-packages".to_string()
} else {
unimplemented!("Only Windows and Unix are supported")
};
let relative_site_packages = pathdiff::diff_paths(
&interpreter.virtualenv().purelib,
&interpreter.virtualenv().scripts,
)
.expect("Failed to calculate relative path to site-packages");
let activator = template
.replace(
"{{ VIRTUAL_ENV_DIR }}",
@ -230,7 +223,7 @@ pub fn create_bare_venv(
)
.replace(
"{{ RELATIVE_SITE_PACKAGES }}",
relative_site_packages.as_str(),
relative_site_packages.simplified().to_str().unwrap(),
);
fs::write(scripts.join(name), activator)?;
}
@ -296,21 +289,7 @@ pub fn create_bare_venv(
drop(pyvenv_cfg);
// Construct the path to the `site-packages` directory.
let site_packages = if cfg!(unix) {
location
.join("lib")
.join(format!(
"{}{}.{}",
interpreter.site_packages_python(),
interpreter.python_major(),
interpreter.python_minor(),
))
.join("site-packages")
} else if cfg!(windows) {
location.join("Lib").join("site-packages")
} else {
unimplemented!("Only Windows and Unix are supported")
};
let site_packages = location.join(&interpreter.virtualenv().purelib);
// Populate `site-packages` with a `_virtualenv.py` file.
fs::create_dir_all(&site_packages)?;
@ -319,15 +298,11 @@ pub fn create_bare_venv(
Ok(Virtualenv {
scheme: Scheme {
// Paths that were already constructed above.
scripts,
// Set `purelib` and `platlib` to the same value.
purelib: site_packages.clone(),
platlib: site_packages,
// Inherited from the interpreter.
stdlib: interpreter.stdlib().to_path_buf(),
include: interpreter.include().to_path_buf(),
data: location.clone(),
purelib: location.join(&interpreter.virtualenv().purelib),
platlib: location.join(&interpreter.virtualenv().platlib),
scripts: location.join(&interpreter.virtualenv().scripts),
data: location.join(&interpreter.virtualenv().data),
include: location.join(&interpreter.virtualenv().include),
},
root: location,
executable,