feat(venv): add relocatable flag (#5515)

## Summary

Adds a `--relocatable` CLI arg to `uv venv`. This flag does two things:

* ensures that the associated activation scripts do not rely on a
hardcoded
absolute path to the virtual environment (to the extent possible; `.csh`
and
  `.nu` left as-is)
* persists a `relocatable` flag in `pyvenv.cfg`.

The flag in `pyvenv.cfg` in turn instructs the wheel `Installer` to
create script
entrypoints in a relocatable way (use `exec` trick + `dirname $0` on
POSIX;
use relative path to `python[w].exe` on Windows).

Fixes: #3863

## Test Plan

* Relocatable console scripts covered as additional scenarios in
existing test cases.
* Integration testing of boilerplate generation in `venv`.
* Manual testing of `uv venv` with and without `--relocatable`
This commit is contained in:
Pavel Dikov 2024-07-29 01:10:11 +01:00 committed by GitHub
parent 3626d08cca
commit cb47aed9de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 299 additions and 26 deletions

View file

@ -1,7 +1,6 @@
//! Takes a wheel and installs it into a venv.
use std::io;
use std::path::PathBuf;
use platform_info::PlatformInfoError;

View file

@ -37,6 +37,7 @@ pub struct Locks(Mutex<FxHashMap<PathBuf, Arc<Mutex<()>>>>);
#[instrument(skip_all, fields(wheel = %filename))]
pub fn install_wheel(
layout: &Layout,
relocatable: bool,
wheel: impl AsRef<Path>,
filename: &WheelFilename,
direct_url: Option<&DirectUrl>,
@ -97,8 +98,22 @@ pub fn install_wheel(
debug!(?name, "Writing entrypoints");
fs_err::create_dir_all(&layout.scheme.scripts)?;
write_script_entrypoints(layout, site_packages, &console_scripts, &mut record, false)?;
write_script_entrypoints(layout, site_packages, &gui_scripts, &mut record, true)?;
write_script_entrypoints(
layout,
relocatable,
site_packages,
&console_scripts,
&mut record,
false,
)?;
write_script_entrypoints(
layout,
relocatable,
site_packages,
&gui_scripts,
&mut record,
true,
)?;
}
// 2.a Unpacked archive includes distribution-1.0.dist-info/ and (if there is data) distribution-1.0.data/.
@ -108,6 +123,7 @@ pub fn install_wheel(
debug!(?name, "Installing data");
install_data(
layout,
relocatable,
site_packages,
&data_dir,
&name,

View file

@ -1,5 +1,5 @@
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Cursor, Read, Write};
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::{env, io};
@ -128,7 +128,7 @@ fn copy_and_hash(reader: &mut impl Read, writer: &mut impl Write) -> io::Result<
/// executable.
///
/// See: <https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_vendor/distlib/scripts.py#L136-L165>
fn format_shebang(executable: impl AsRef<Path>, os_name: &str) -> String {
fn format_shebang(executable: impl AsRef<Path>, os_name: &str, relocatable: bool) -> String {
// Convert the executable to a simplified path.
let executable = executable.as_ref().simplified_display().to_string();
@ -139,11 +139,18 @@ fn format_shebang(executable: impl AsRef<Path>, os_name: &str) -> String {
let shebang_length = 2 + executable.len() + 1;
// If the shebang is too long, or contains spaces, wrap it in `/bin/sh`.
if shebang_length > 127 || executable.contains(' ') {
// Same applies for relocatable scripts (executable is relative to script dir, hence `dirname` trick)
// (note: the Windows trampoline binaries natively support relative paths to executable)
if shebang_length > 127 || executable.contains(' ') || relocatable {
let prefix = if relocatable {
r#""$(CDPATH= cd -- "$(dirname -- "$0")" && echo "$PWD")"/"#
} else {
""
};
// Like Python's `shlex.quote`:
// > Use single quotes, and put single quotes into double quotes
// > The string $'b is then quoted as '$'"'"'b'
let executable = format!("'{}'", executable.replace('\'', r#"'"'"'"#));
let executable = format!("{}'{}'", prefix, executable.replace('\'', r#"'"'"'"#));
return format!("#!/bin/sh\n'''exec' {executable} \"$0\" \"$@\"\n' '''");
}
}
@ -272,6 +279,7 @@ fn entrypoint_path(entrypoint: &Script, layout: &Layout) -> PathBuf {
/// Create the wrapper scripts in the bin folder of the venv for launching console scripts.
pub(crate) fn write_script_entrypoints(
layout: &Layout,
relocatable: bool,
site_packages: &Path,
entrypoints: &[Script],
record: &mut Vec<RecordEntry>,
@ -293,9 +301,11 @@ pub(crate) fn write_script_entrypoints(
// Generate the launcher script.
let launcher_executable = get_script_executable(&layout.sys_executable, is_gui);
let launcher_executable =
get_relocatable_executable(launcher_executable, layout, relocatable)?;
let launcher_python_script = get_script_launcher(
entrypoint,
&format_shebang(&launcher_executable, &layout.os_name),
&format_shebang(&launcher_executable, &layout.os_name, relocatable),
);
// If necessary, wrap the launcher script in a Windows launcher binary.
@ -432,11 +442,12 @@ pub(crate) fn move_folder_recorded(
Ok(())
}
/// Installs a single script (not an entrypoint)
/// Installs a single script (not an entrypoint).
///
/// Has to deal with both binaries files (just move) and scripts (rewrite the shebang if applicable)
/// Has to deal with both binaries files (just move) and scripts (rewrite the shebang if applicable).
fn install_script(
layout: &Layout,
relocatable: bool,
site_packages: &Path,
record: &mut [RecordEntry],
file: &DirEntry,
@ -494,7 +505,19 @@ fn install_script(
let mut start = vec![0; placeholder_python.len()];
script.read_exact(&mut start)?;
let size_and_encoded_hash = if start == placeholder_python {
let start = format_shebang(&layout.sys_executable, &layout.os_name)
let is_gui = {
let mut buf = vec![0; 1];
script.read_exact(&mut buf)?;
if buf == b"w" {
true
} else {
script.seek_relative(-1)?;
false
}
};
let executable = get_script_executable(&layout.sys_executable, is_gui);
let executable = get_relocatable_executable(executable, layout, relocatable)?;
let start = format_shebang(&executable, &layout.os_name, relocatable)
.as_bytes()
.to_vec();
@ -555,6 +578,7 @@ fn install_script(
#[instrument(skip_all)]
pub(crate) fn install_data(
layout: &Layout,
relocatable: bool,
site_packages: &Path,
data_dir: &Path,
dist_name: &PackageName,
@ -598,7 +622,7 @@ pub(crate) fn install_data(
initialized = true;
}
install_script(layout, site_packages, record, &file)?;
install_script(layout, relocatable, site_packages, record, &file)?;
}
}
Some("headers") => {
@ -682,6 +706,31 @@ pub(crate) fn extra_dist_info(
Ok(())
}
/// Get the path to the Python executable for the [`Layout`], based on whether the wheel should
/// be relocatable.
///
/// Returns `sys.executable` if the wheel is not relocatable; otherwise, returns a path relative
/// to the scripts directory.
pub(crate) fn get_relocatable_executable(
executable: PathBuf,
layout: &Layout,
relocatable: bool,
) -> Result<PathBuf, Error> {
Ok(if relocatable {
pathdiff::diff_paths(&executable, &layout.scheme.scripts).ok_or_else(|| {
Error::Io(io::Error::new(
io::ErrorKind::Other,
format!(
"Could not find relative path for: {}",
executable.simplified_display()
),
))
})?
} else {
executable
})
}
/// Reads the record file
/// <https://www.python.org/dev/peps/pep-0376/#record>
pub fn read_record_file(record: &mut impl Read) -> Result<Vec<RecordEntry>, Error> {
@ -845,33 +894,47 @@ mod test {
// By default, use a simple shebang.
let executable = Path::new("/usr/bin/python3");
let os_name = "posix";
assert_eq!(format_shebang(executable, os_name), "#!/usr/bin/python3");
assert_eq!(
format_shebang(executable, os_name, false),
"#!/usr/bin/python3"
);
// If the path contains spaces, we should use the `exec` trick.
let executable = Path::new("/usr/bin/path to python3");
let os_name = "posix";
assert_eq!(
format_shebang(executable, os_name),
format_shebang(executable, os_name, false),
"#!/bin/sh\n'''exec' '/usr/bin/path to python3' \"$0\" \"$@\"\n' '''"
);
// And if we want a relocatable script, we should use the `exec` trick with `dirname`.
let executable = Path::new("python3");
let os_name = "posix";
assert_eq!(
format_shebang(executable, os_name, true),
"#!/bin/sh\n'''exec' \"$(CDPATH= cd -- \"$(dirname -- \"$0\")\" && echo \"$PWD\")\"/'python3' \"$0\" \"$@\"\n' '''"
);
// Except on Windows...
let executable = Path::new("/usr/bin/path to python3");
let os_name = "nt";
assert_eq!(
format_shebang(executable, os_name),
format_shebang(executable, os_name, false),
"#!/usr/bin/path to python3"
);
// Quotes, however, are ok.
let executable = Path::new("/usr/bin/'python3'");
let os_name = "posix";
assert_eq!(format_shebang(executable, os_name), "#!/usr/bin/'python3'");
assert_eq!(
format_shebang(executable, os_name, false),
"#!/usr/bin/'python3'"
);
// If the path is too long, we should not use the `exec` trick.
let executable = Path::new("/usr/bin/path/to/a/very/long/executable/executable/executable/executable/executable/executable/executable/executable/name/python3");
let os_name = "posix";
assert_eq!(format_shebang(executable, os_name), "#!/bin/sh\n'''exec' '/usr/bin/path/to/a/very/long/executable/executable/executable/executable/executable/executable/executable/executable/name/python3' \"$0\" \"$@\"\n' '''");
assert_eq!(format_shebang(executable, os_name, false), "#!/bin/sh\n'''exec' '/usr/bin/path/to/a/very/long/executable/executable/executable/executable/executable/executable/executable/executable/name/python3' \"$0\" \"$@\"\n' '''");
}
#[test]

View file

@ -442,6 +442,7 @@ impl SourceBuild {
uv_virtualenv::Prompt::None,
false,
false,
false,
)?,
BuildIsolation::Shared(venv) => venv.clone(),
};

View file

@ -1743,6 +1743,22 @@ pub struct VenvArgs {
#[arg(long)]
pub system_site_packages: bool,
/// Make the virtual environment relocatable.
///
/// A relocatable virtual environment can be moved around and redistributed without
/// invalidating its associated entrypoint and activation scripts.
///
/// Note that this can only be guaranteed for standard `console_scripts` and `gui_scripts`.
/// Other scripts may be adjusted if they ship with a generic `#!python[w]` shebang,
/// and binaries are left as-is.
///
/// As a result of making the environment relocatable (by way of writing relative, rather than
/// absolute paths), the entrypoints and scripts themselves will _not_ be relocatable. In other
/// words, copying those entrypoints and scripts to a location outside the environment will not
/// work, as they reference paths relative to the environment itself.
#[arg(long)]
pub relocatable: bool,
#[command(flatten)]
pub index_args: IndexArgs,

View file

@ -85,8 +85,16 @@ impl<'a> Installer<'a> {
let (tx, rx) = oneshot::channel();
let layout = venv.interpreter().layout();
let relocatable = venv.relocatable();
rayon::spawn(move || {
let result = install(wheels, layout, installer_name, link_mode, reporter);
let result = install(
wheels,
layout,
installer_name,
link_mode,
reporter,
relocatable,
);
tx.send(result).unwrap();
});
@ -112,6 +120,7 @@ impl<'a> Installer<'a> {
self.installer_name,
self.link_mode,
self.reporter,
self.venv.relocatable(),
)
}
}
@ -124,11 +133,13 @@ fn install(
installer_name: Option<String>,
link_mode: LinkMode,
reporter: Option<Box<dyn Reporter>>,
relocatable: bool,
) -> Result<Vec<CachedDist>> {
let locks = install_wheel_rs::linker::Locks::default();
wheels.par_iter().try_for_each(|wheel| {
install_wheel_rs::linker::install_wheel(
&layout,
relocatable,
wheel.path(),
wheel.filename(),
wheel

View file

@ -162,6 +162,11 @@ impl PythonEnvironment {
Ok(PyVenvConfiguration::parse(self.0.root.join("pyvenv.cfg"))?)
}
/// Returns `true` if the environment is "relocatable".
pub fn relocatable(&self) -> bool {
self.cfg().is_ok_and(|cfg| cfg.is_relocatable())
}
/// Returns the location of the Python executable.
pub fn python_executable(&self) -> &Path {
self.0.interpreter.sys_executable()

View file

@ -28,6 +28,8 @@ pub struct PyVenvConfiguration {
pub(crate) virtualenv: bool,
/// If the uv package was used to create the virtual environment.
pub(crate) uv: bool,
/// Is the virtual environment relocatable?
pub(crate) relocatable: bool,
}
#[derive(Debug, Error)]
@ -136,6 +138,7 @@ impl PyVenvConfiguration {
pub fn parse(cfg: impl AsRef<Path>) -> Result<Self, Error> {
let mut virtualenv = false;
let mut uv = false;
let mut relocatable = false;
// Per https://snarky.ca/how-virtual-environments-work/, the `pyvenv.cfg` file is not a
// valid INI file, and is instead expected to be parsed by partitioning each line on the
@ -143,7 +146,7 @@ impl PyVenvConfiguration {
let content = fs::read_to_string(&cfg)
.map_err(|err| Error::ParsePyVenvCfg(cfg.as_ref().to_path_buf(), err))?;
for line in content.lines() {
let Some((key, _value)) = line.split_once('=') else {
let Some((key, value)) = line.split_once('=') else {
continue;
};
match key.trim() {
@ -153,11 +156,18 @@ impl PyVenvConfiguration {
"uv" => {
uv = true;
}
"relocatable" => {
relocatable = value.trim().to_lowercase() == "true";
}
_ => {}
}
}
Ok(Self { virtualenv, uv })
Ok(Self {
virtualenv,
uv,
relocatable,
})
}
/// Returns true if the virtual environment was created with the `virtualenv` package.
@ -169,4 +179,9 @@ impl PyVenvConfiguration {
pub fn is_uv(&self) -> bool {
self.uv
}
/// Returns true if the virtual environment is relocatable.
pub fn is_relocatable(&self) -> bool {
self.relocatable
}
}

View file

@ -258,6 +258,7 @@ impl InstalledTools {
uv_virtualenv::Prompt::None,
false,
false,
false,
)?;
Ok(venv)

View file

@ -19,7 +19,7 @@
@REM OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
@REM WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@set "VIRTUAL_ENV={{ VIRTUAL_ENV_DIR }}"
@for %%i in ("{{ VIRTUAL_ENV_DIR }}") do @set "VIRTUAL_ENV=%%~fi"
@set "VIRTUAL_ENV_PROMPT={{ VIRTUAL_PROMPT }}"
@if NOT DEFINED VIRTUAL_ENV_PROMPT (

View file

@ -52,6 +52,7 @@ pub fn create_venv(
prompt: Prompt,
system_site_packages: bool,
allow_existing: bool,
relocatable: bool,
) -> Result<PythonEnvironment, Error> {
// Create the virtualenv at the given location.
let virtualenv = virtualenv::create(
@ -60,6 +61,7 @@ pub fn create_venv(
prompt,
system_site_packages,
allow_existing,
relocatable,
)?;
// Create the corresponding `PythonEnvironment`.

View file

@ -50,6 +50,7 @@ pub(crate) fn create(
prompt: Prompt,
system_site_packages: bool,
allow_existing: bool,
relocatable: bool,
) -> Result<VirtualEnvironment, Error> {
// Determine the base Python executable; that is, the Python executable that should be
// considered the "base" for the virtual environment. This is typically the Python executable
@ -294,12 +295,27 @@ pub(crate) fn create(
.map(|path| path.simplified().to_str().unwrap().replace('\\', "\\\\"))
.join(path_sep);
let activator = template
.replace(
"{{ VIRTUAL_ENV_DIR }}",
let virtual_env_dir = match (relocatable, name.to_owned()) {
(true, "activate") => {
// Extremely verbose, but should cover all major POSIX shells,
// as well as platforms where `readlink` does not implement `-f`.
r#"'"$(dirname -- "$(CDPATH= cd -- "$(dirname -- ${BASH_SOURCE[0]:-${(%):-%x}})" && echo "$PWD")")"'"#
}
(true, "activate.bat") => r"%~dp0..",
(true, "activate.fish") => {
r#"'"$(dirname -- "$(cd "$(dirname -- "$(status -f)")"; and pwd)")"'"#
}
// Note:
// * relocatable activate scripts appear not to be possible in csh and nu shell
// * `activate.ps1` is already relocatable by default.
_ => {
// SAFETY: `unwrap` is guaranteed to succeed because `location` is an `Utf8PathBuf`.
location.simplified().to_str().unwrap(),
)
location.simplified().to_str().unwrap()
}
};
let activator = template
.replace("{{ VIRTUAL_ENV_DIR }}", virtual_env_dir)
.replace("{{ BIN_NAME }}", bin_name)
.replace(
"{{ VIRTUAL_PROMPT }}",
@ -335,6 +351,14 @@ pub(crate) fn create(
"false".to_string()
},
),
(
"relocatable".to_string(),
if relocatable {
"true".to_string()
} else {
"false".to_string()
},
),
];
if let Some(prompt) = prompt {

View file

@ -132,6 +132,7 @@ impl CachedEnvironment {
uv_virtualenv::Prompt::None,
false,
false,
false,
)?;
let venv = sync_environment(

View file

@ -313,6 +313,7 @@ pub(crate) async fn get_or_init_environment(
uv_virtualenv::Prompt::None,
false,
false,
false,
)?)
}
}

View file

@ -385,6 +385,7 @@ pub(crate) async fn run(
uv_virtualenv::Prompt::None,
false,
false,
false,
)?;
match spec {

View file

@ -28,6 +28,7 @@ use uv_python::{
use uv_resolver::{ExcludeNewer, FlatIndex};
use uv_shell::Shell;
use uv_types::{BuildContext, BuildIsolation, HashStrategy};
use uv_warnings::warn_user_once;
use crate::commands::reporters::PythonDownloadReporter;
use crate::commands::{pip, ExitStatus, SharedState};
@ -54,6 +55,7 @@ pub(crate) async fn venv(
preview: PreviewMode,
cache: &Cache,
printer: Printer,
relocatable: bool,
) -> Result<ExitStatus> {
match venv_impl(
path,
@ -74,6 +76,7 @@ pub(crate) async fn venv(
native_tls,
cache,
printer,
relocatable,
)
.await
{
@ -125,6 +128,7 @@ async fn venv_impl(
native_tls: bool,
cache: &Cache,
printer: Printer,
relocatable: bool,
) -> miette::Result<ExitStatus> {
let client_builder = BaseClientBuilder::default()
.connectivity(connectivity)
@ -138,6 +142,9 @@ async fn venv_impl(
if preview.is_enabled() && interpreter_request.is_none() {
interpreter_request = request_from_version_file().await.into_diagnostic()?;
}
if preview.is_disabled() && relocatable {
warn_user_once!("`--relocatable` is experimental and may change without warning");
}
// Locate the Python interpreter to use in the environment
let python = PythonInstallation::find_or_fetch(
@ -192,6 +199,7 @@ async fn venv_impl(
prompt,
system_site_packages,
allow_existing,
relocatable,
)
.map_err(VenvError::Creation)?;

View file

@ -593,6 +593,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
globals.preview,
&cache,
printer,
args.relocatable,
)
.await
}

View file

@ -1407,6 +1407,7 @@ pub(crate) struct VenvSettings {
pub(crate) name: PathBuf,
pub(crate) prompt: Option<String>,
pub(crate) system_site_packages: bool,
pub(crate) relocatable: bool,
pub(crate) settings: PipSettings,
}
@ -1422,6 +1423,7 @@ impl VenvSettings {
name,
prompt,
system_site_packages,
relocatable,
index_args,
index_strategy,
keyring_provider,
@ -1436,6 +1438,7 @@ impl VenvSettings {
name,
prompt,
system_site_packages,
relocatable,
settings: PipSettings::combine(
PipOptions {
python,

View file

@ -6,8 +6,10 @@ use anyhow::Result;
use assert_cmd::prelude::*;
use assert_fs::prelude::*;
use base64::{prelude::BASE64_STANDARD as base64, Engine};
use fs_err as fs;
use indoc::indoc;
use itertools::Itertools;
use predicates::prelude::predicate;
use url::Url;
use common::{uv_snapshot, TestContext};
@ -6188,3 +6190,57 @@ fn unmanaged() -> Result<()> {
Ok(())
}
#[test]
fn install_relocatable() -> Result<()> {
let context = TestContext::new("3.12");
// Remake the venv as relocatable
context
.venv()
.arg(context.venv.as_os_str())
.arg("--python")
.arg("3.12")
.arg("--relocatable")
.assert()
.success();
// Install a package with a hello-world console script entrypoint.
// (we use black_editable because it's convenient, but we don't actually install it as editable)
context
.pip_install()
.arg(
context
.workspace_root
.join("scripts/packages/black_editable"),
)
.assert()
.success();
// Script should run correctly in-situ.
let script_path = if cfg!(windows) {
context.venv.child(r"Scripts\black.exe")
} else {
context.venv.child("bin/black")
};
Command::new(script_path.as_os_str())
.assert()
.success()
.stdout(predicate::str::contains("Hello world!"));
// Relocate the venv, and see if it still works.
let new_venv_path = context.venv.with_file_name("relocated");
fs::rename(context.venv, new_venv_path.clone())?;
let script_path = if cfg!(windows) {
new_venv_path.join(r"Scripts\black.exe")
} else {
new_venv_path.join("bin/black")
};
Command::new(script_path.as_os_str())
.assert()
.success()
.stdout(predicate::str::contains("Hello world!"));
Ok(())
}

View file

@ -577,6 +577,55 @@ fn verify_pyvenv_cfg() {
let version = env!("CARGO_PKG_VERSION").to_string();
let search_string = format!("uv = {version}");
pyvenv_cfg.assert(predicates::str::contains(search_string));
// Not relocatable by default.
pyvenv_cfg.assert(predicates::str::contains("relocatable = false"));
}
#[test]
fn verify_pyvenv_cfg_relocatable() {
let context = TestContext::new("3.12");
// Create a virtual environment at `.venv`.
context
.venv()
.arg(context.venv.as_os_str())
.arg("--python")
.arg("3.12")
.arg("--relocatable")
.assert()
.success();
let pyvenv_cfg = context.venv.child("pyvenv.cfg");
context.venv.assert(predicates::path::is_dir());
// Check pyvenv.cfg exists
pyvenv_cfg.assert(predicates::path::is_file());
// Relocatable flag is set.
pyvenv_cfg.assert(predicates::str::contains("relocatable = true"));
// Activate scripts contain the relocatable boilerplate
let scripts = if cfg!(windows) {
context.venv.child("Scripts")
} else {
context.venv.child("bin")
};
let activate_sh = scripts.child("activate");
activate_sh.assert(predicates::path::is_file());
activate_sh.assert(predicates::str::contains(r#"VIRTUAL_ENV=''"$(dirname -- "$(CDPATH= cd -- "$(dirname -- ${BASH_SOURCE[0]:-${(%):-%x}})" && echo "$PWD")")"''"#));
let activate_bat = scripts.child("activate.bat");
activate_bat.assert(predicates::path::is_file());
activate_bat.assert(predicates::str::contains(
r#"@for %%i in ("%~dp0..") do @set "VIRTUAL_ENV=%%~fi""#,
));
let activate_fish = scripts.child("activate.fish");
activate_fish.assert(predicates::path::is_file());
activate_fish.assert(predicates::str::contains(r#"set -gx VIRTUAL_ENV ''"$(dirname -- "$(cd "$(dirname -- "$(status -f)")"; and pwd)")"''"#));
}
/// Ensure that a nested virtual environment uses the same `home` directory as the parent.