Query interpreter to determine correct virtualenv paths (#2188)

## Summary

This PR migrates our virtualenv creation from a setup that assumes prior
knowledge of the correct paths, to a technique borrowed from
`virtualenv` whereby we use `sysconfig` and `distutils` to determine the
paths. The general trick is to grab the expected paths with `sysconfig`,
then make them all relative, then make them absolute for a given
directory.

Closes #2095.
Closes #2153.

Cargo.lock

@@ -4662,6 +4662,7 @@ dependencies = [
  "clap",
  "directories",
  "fs-err",
+ "pathdiff",
  "platform-host",
  "pypi-types",
  "serde",

crates/pypi-types/src/scheme.rs

@@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize};
 /// See: <https://docs.python.org/3.12/library/sysconfig.html#installation-paths>
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct Scheme {
-    pub stdlib: PathBuf,
     pub purelib: PathBuf,
     pub platlib: PathBuf,
     pub scripts: PathBuf,

crates/uv-interpreter/src/get_interpreter_info.py

@@ -7,7 +7,6 @@ Exit Codes:
     3: Python version 3 or newer is required
 """
 
-
 import json
 import os
 import platform
@@ -67,9 +66,126 @@ python_full_version = platform.python_version()
 # [3]: https://github.com/pypa/packaging/issues/678#issuecomment-1436033646
 # [4]: https://github.com/astral-sh/uv/issues/1357#issuecomment-1947645243
 # [5]: https://github.com/pypa/packaging/blob/085ff41692b687ae5b0772a55615b69a5b677be9/packaging/version.py#L168-L193
-if len(python_full_version) > 0 and python_full_version[-1] == '+':
+if len(python_full_version) > 0 and python_full_version[-1] == "+":
     python_full_version = python_full_version[:-1]
 
+
+def get_virtualenv():
+    """Return the expected Scheme for virtualenvs created by this interpreter.
+
+    The paths returned should be relative to a root directory.
+
+    This is based on virtualenv's path discovery logic:
+        https://github.com/pypa/virtualenv/blob/5cd543fdf8047600ff2737babec4a635ad74d169/src/virtualenv/discovery/py_info.py#L80C9-L80C17
+    """
+    scheme_names = sysconfig.get_scheme_names()
+
+    # Determine the scheme to use, if any.
+    if "venv" in scheme_names:
+        sysconfig_scheme = "venv"
+    elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names:
+        # debian / ubuntu python 3.10 without `python3-distutils` will report
+        # mangled `local/bin` / etc. names for the default prefix
+        # intentionally select `posix_prefix` which is the unaltered posix-like paths
+        sysconfig_scheme = "posix_prefix"
+    else:
+        sysconfig_scheme = None
+
+    # Use `sysconfig`, if available.
+    if sysconfig_scheme:
+        import re
+
+        sysconfig_paths = {
+            i: sysconfig.get_path(i, expand=False, scheme=sysconfig_scheme)
+            for i in sysconfig.get_path_names()
+        }
+
+        # Determine very configuration variable that we need to resolve.
+        config_var_keys = set()
+
+        conf_var_re = re.compile(r"\{\w+}")
+        for element in sysconfig_paths.values():
+            for k in conf_var_re.findall(element):
+                config_var_keys.add(k[1:-1])
+        config_var_keys.add("PYTHONFRAMEWORK")
+
+        # Look them up.
+        sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys}
+
+        # Information about the prefix (determines the Python home).
+        prefix = os.path.abspath(sys.prefix)
+        base_prefix = os.path.abspath(sys.base_prefix)
+
+        # Information about the exec prefix (dynamic stdlib modules).
+        base_exec_prefix = os.path.abspath(sys.base_exec_prefix)
+        exec_prefix = os.path.abspath(sys.exec_prefix)
+
+        # Set any prefixes to empty, which makes the resulting paths relative.
+        prefixes = prefix, exec_prefix, base_prefix, base_exec_prefix
+        sysconfig_vars.update(
+            {k: "" if v in prefixes else v for k, v in sysconfig_vars.items()}
+        )
+
+        def expand_path(path: str) -> str:
+            return path.format(**sysconfig_vars).replace("/", os.sep).lstrip(os.sep)
+
+        return {
+            "purelib": expand_path(sysconfig_paths["purelib"]),
+            "platlib": expand_path(sysconfig_paths["platlib"]),
+            "include": expand_path(sysconfig_paths["include"]),
+            "scripts": expand_path(sysconfig_paths["scripts"]),
+            "data": expand_path(sysconfig_paths["data"]),
+        }
+    else:
+        # Use distutils primarily because that's what pip does.
+        # https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L249
+        import warnings
+
+        with warnings.catch_warnings():  # disable warning for PEP-632
+            warnings.simplefilter("ignore")
+            from distutils import dist
+            from distutils.command.install import SCHEME_KEYS
+
+        d = dist.Distribution({"script_args": "--no-user-cfg"})
+        if hasattr(sys, "_framework"):
+            sys._framework = None
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            i = d.get_command_obj("install", create=True)
+
+        i.prefix = os.sep
+        i.finalize_options()
+        distutils_paths = {
+            key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep)
+            for key in SCHEME_KEYS
+        }
+
+        return {
+            "purelib": distutils_paths["purelib"],
+            "platlib": distutils_paths["platlib"],
+            "include": os.path.dirname(distutils_paths["headers"]),
+            "scripts": distutils_paths["scripts"],
+            "data": distutils_paths["data"],
+        }
+
+
+def get_scheme():
+    """Return the Scheme for the current interpreter.
+
+    The paths returned should be absolute.
+    """
+    # TODO(charlie): Use distutils on required Python distributions.
+    paths = sysconfig.get_paths()
+    return {
+        "purelib": paths["purelib"],
+        "platlib": paths["platlib"],
+        "include": paths["include"],
+        "scripts": paths["scripts"],
+        "data": paths["data"],
+    }
+
+
 markers = {
     "implementation_name": implementation_name,
     "implementation_version": implementation_version,
@@ -90,6 +206,8 @@ interpreter_info = {
     "prefix": sys.prefix,
     "base_executable": getattr(sys, "_base_executable", None),
     "sys_executable": sys.executable,
-    "scheme": sysconfig.get_paths(),
+    "stdlib": sysconfig.get_path("stdlib"),
+    "scheme": get_scheme(),
+    "virtualenv": get_virtualenv(),
 }
 print(json.dumps(interpreter_info))

crates/uv-interpreter/src/interpreter.rs

@@ -28,11 +28,13 @@ pub struct Interpreter {
     platform: Platform,
     markers: Box<MarkerEnvironment>,
     scheme: Scheme,
+    virtualenv: Scheme,
     prefix: PathBuf,
     base_exec_prefix: PathBuf,
     base_prefix: PathBuf,
     base_executable: Option<PathBuf>,
     sys_executable: PathBuf,
+    stdlib: PathBuf,
     tags: OnceCell<Tags>,
 }
 
@@ -51,11 +53,13 @@ impl Interpreter {
             platform,
             markers: Box::new(info.markers),
             scheme: info.scheme,
+            virtualenv: info.virtualenv,
             prefix: info.prefix,
             base_exec_prefix: info.base_exec_prefix,
             base_prefix: info.base_prefix,
             base_executable: info.base_executable,
             sys_executable: info.sys_executable,
+            stdlib: info.stdlib,
             tags: OnceCell::new(),
         })
     }
@@ -66,7 +70,13 @@ impl Interpreter {
             platform,
             markers: Box::new(markers),
             scheme: Scheme {
-                stdlib: PathBuf::from("/dev/null"),
+                purelib: PathBuf::from("/dev/null"),
+                platlib: PathBuf::from("/dev/null"),
+                include: PathBuf::from("/dev/null"),
+                scripts: PathBuf::from("/dev/null"),
+                data: PathBuf::from("/dev/null"),
+            },
+            virtualenv: Scheme {
                 purelib: PathBuf::from("/dev/null"),
                 platlib: PathBuf::from("/dev/null"),
                 include: PathBuf::from("/dev/null"),
@@ -78,6 +88,7 @@ impl Interpreter {
             base_prefix: PathBuf::from("/dev/null"),
             base_executable: None,
             sys_executable: PathBuf::from("/dev/null"),
+            stdlib: PathBuf::from("/dev/null"),
             tags: OnceCell::new(),
         }
     }
@@ -240,7 +251,7 @@ impl Interpreter {
             return None;
         }
 
-        let Ok(contents) = fs::read_to_string(self.scheme.stdlib.join("EXTERNALLY-MANAGED")) else {
+        let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
             return None;
         };
 
@@ -345,6 +356,11 @@ impl Interpreter {
         &self.sys_executable
     }
 
+    /// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
+    pub fn stdlib(&self) -> &Path {
+        &self.stdlib
+    }
+
     /// Return the `purelib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
     pub fn purelib(&self) -> &Path {
         &self.scheme.purelib
@@ -370,21 +386,9 @@ impl Interpreter {
         &self.scheme.include
     }
 
-    /// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
+    /// Return the [`Scheme`] for a virtual environment created by this [`Interpreter`].
-    pub fn stdlib(&self) -> &Path {
+    pub fn virtualenv(&self) -> &Scheme {
-        &self.scheme.stdlib
+        &self.virtualenv
-    }
-
-    /// Return the name of the Python directory used to build the path to the
-    /// `site-packages` directory.
-    ///
-    /// If one could not be determined, then `python` is returned.
-    pub fn site_packages_python(&self) -> &str {
-        if self.implementation_name() == "pypy" {
-            "pypy"
-        } else {
-            "python"
-        }
     }
 
     /// Return the [`Layout`] environment used to install wheels into this interpreter.
@@ -394,7 +398,6 @@ impl Interpreter {
             sys_executable: self.sys_executable().to_path_buf(),
             os_name: self.markers.os_name.clone(),
             scheme: Scheme {
-                stdlib: self.stdlib().to_path_buf(),
                 purelib: self.purelib().to_path_buf(),
                 platlib: self.platlib().to_path_buf(),
                 scripts: self.scripts().to_path_buf(),
@@ -403,8 +406,7 @@ impl Interpreter {
                     // If the interpreter is a venv, then the `include` directory has a different structure.
                     // See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
                     self.prefix.join("include").join("site").join(format!(
-                        "{}{}.{}",
+                        "python{}.{}",
-                        self.site_packages_python(),
                         self.python_major(),
                         self.python_minor()
                     ))
@@ -435,11 +437,13 @@ impl ExternallyManaged {
 struct InterpreterInfo {
     markers: MarkerEnvironment,
     scheme: Scheme,
+    virtualenv: Scheme,
     prefix: PathBuf,
     base_exec_prefix: PathBuf,
     base_prefix: PathBuf,
     base_executable: Option<PathBuf>,
     sys_executable: PathBuf,
+    stdlib: PathBuf,
 }
 
 impl InterpreterInfo {
@@ -635,13 +639,20 @@ mod tests {
                 "base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
                 "prefix": "/home/ferris/projects/uv/.venv",
                 "sys_executable": "/home/ferris/projects/uv/.venv/bin/python",
+                "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12",
                 "scheme": {
                     "data": "/home/ferris/.pyenv/versions/3.12.0",
                     "include": "/home/ferris/.pyenv/versions/3.12.0/include",
                     "platlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
                     "purelib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
-                    "scripts": "/home/ferris/.pyenv/versions/3.12.0/bin",
+                    "scripts": "/home/ferris/.pyenv/versions/3.12.0/bin"
-                    "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12"
+                },
+                "virtualenv": {
+                    "data": "",
+                    "include": "include",
+                    "platlib": "lib/python3.12/site-packages",
+                    "purelib": "lib/python3.12/site-packages",
+                    "scripts": "bin"
                 }
             }
         "##};

crates/uv-virtualenv/Cargo.toml

@@ -32,6 +32,7 @@ cachedir = { workspace = true }
 clap = { workspace = true, features = ["derive"], optional = true }
 directories = { workspace = true }
 fs-err = { workspace = true }
+pathdiff = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tempfile = { workspace = true }

crates/uv-virtualenv/src/bare.rs

@@ -8,9 +8,9 @@ use std::path::Path;
 
 use fs_err as fs;
 use fs_err::File;
+use pypi_types::Scheme;
 use tracing::info;
 
-use pypi_types::Scheme;
 use uv_fs::Simplified;
 use uv_interpreter::{Interpreter, Virtualenv};
 
@@ -124,7 +124,7 @@ pub fn create_bare_venv(
     } else {
         unimplemented!("Only Windows and Unix are supported")
     };
-    let scripts = location.join(bin_name);
+    let scripts = location.join(&interpreter.virtualenv().scripts);
     let prompt = match prompt {
         Prompt::CurrentDirectoryName => env::current_dir()?
             .file_name()
@@ -143,7 +143,6 @@ pub fn create_bare_venv(
     fs::create_dir(&scripts)?;
     let executable = scripts.join(format!("python{EXE_SUFFIX}"));
 
-    // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
     #[cfg(unix)]
     {
         use fs_err::os::unix::fs::symlink;
@@ -163,6 +162,7 @@ pub fn create_bare_venv(
         )?;
     }
 
+    // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
     #[cfg(windows)]
     {
         // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267
@@ -205,18 +205,11 @@ pub fn create_bare_venv(
 
     // Add all the activate scripts for different shells
     for (name, template) in ACTIVATE_TEMPLATES {
-        let relative_site_packages = if cfg!(unix) {
+        let relative_site_packages = pathdiff::diff_paths(
-            format!(
+            &interpreter.virtualenv().purelib,
-                "../lib/{}{}.{}/site-packages",
+            &interpreter.virtualenv().scripts,
-                interpreter.site_packages_python(),
+        )
-                interpreter.python_major(),
+        .expect("Failed to calculate relative path to site-packages");
-                interpreter.python_minor(),
-            )
-        } else if cfg!(windows) {
-            "../Lib/site-packages".to_string()
-        } else {
-            unimplemented!("Only Windows and Unix are supported")
-        };
         let activator = template
             .replace(
                 "{{ VIRTUAL_ENV_DIR }}",
@@ -230,7 +223,7 @@ pub fn create_bare_venv(
             )
             .replace(
                 "{{ RELATIVE_SITE_PACKAGES }}",
-                relative_site_packages.as_str(),
+                relative_site_packages.simplified().to_str().unwrap(),
             );
         fs::write(scripts.join(name), activator)?;
     }
@@ -296,21 +289,7 @@ pub fn create_bare_venv(
     drop(pyvenv_cfg);
 
     // Construct the path to the `site-packages` directory.
-    let site_packages = if cfg!(unix) {
+    let site_packages = location.join(&interpreter.virtualenv().purelib);
-        location
-            .join("lib")
-            .join(format!(
-                "{}{}.{}",
-                interpreter.site_packages_python(),
-                interpreter.python_major(),
-                interpreter.python_minor(),
-            ))
-            .join("site-packages")
-    } else if cfg!(windows) {
-        location.join("Lib").join("site-packages")
-    } else {
-        unimplemented!("Only Windows and Unix are supported")
-    };
 
     // Populate `site-packages` with a `_virtualenv.py` file.
     fs::create_dir_all(&site_packages)?;
@@ -319,15 +298,11 @@ pub fn create_bare_venv(
 
     Ok(Virtualenv {
         scheme: Scheme {
-            // Paths that were already constructed above.
+            purelib: location.join(&interpreter.virtualenv().purelib),
-            scripts,
+            platlib: location.join(&interpreter.virtualenv().platlib),
-            // Set `purelib` and `platlib` to the same value.
+            scripts: location.join(&interpreter.virtualenv().scripts),
-            purelib: site_packages.clone(),
+            data: location.join(&interpreter.virtualenv().data),
-            platlib: site_packages,
+            include: location.join(&interpreter.virtualenv().include),
-            // Inherited from the interpreter.
-            stdlib: interpreter.stdlib().to_path_buf(),
-            include: interpreter.include().to_path_buf(),
-            data: location.clone(),
         },
         root: location,
         executable,