mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Unify python interpreter abstractions (#178)

Browse source 
Previously, we had two python interpreter metadata structs, one in
gourgeist and one in puffin. Both would spawn a subprocess to query
overlapping metadata and both would appear in the cli crate, if you
weren't careful you could even have to different base interpreters at
once. This change unifies this to one set of metadata, queried and
cached once.

Another effect of this crate is proper separation of python interpreter
and venv. A base interpreter (such as `/usr/bin/python/`, but also pyenv
and conda installed python) has a set of metadata. A venv has a root and
inherits the base python metadata except for `sys.prefix`, which unlike
`sys.base_prefix`, gets set to the venv root. From the root and the
interpreter info we can compute the paths inside the venv. We can reuse
the interpreter info of the base interpreter when creating a venv
without having to query the newly created `python`.
This commit is contained in:
konsti 2023-10-25 22:11:36 +02:00 • committed by GitHub
parent 1fbe328257
commit 889f6173cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 515 additions and 584 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Cargo.lock generated
View file

@ -1003,9 +1003,10 @@ dependencies = [
"distribution-filename",
"fs-err",
"install-wheel-rs",
"platform-host",
"puffin-interpreter",
"rayon",
"reqwest",
"seahash",
"serde",
"serde_json",
"tempfile",
@ -2021,6 +2022,7 @@ dependencies = [
"pep508_rs",
"platform-host",
"platform-tags",
"puffin-interpreter",
"puffin-traits",
"pyproject-toml",
"serde",
@ -2180,7 +2182,9 @@ dependencies = [
"pep440_rs 0.3.12",
"pep508_rs",
"platform-host",
"serde",
"serde_json",
"thiserror",
"tokio",
"tracing",
]
@ -2243,7 +2247,6 @@ dependencies = [
"tracing",
"url",
"waitmap",
"which",
"zip",
]
@ -2252,7 +2255,6 @@ name = "puffin-traits"
version = "0.1.0"
dependencies = [
"anyhow",
"gourgeist",
"pep508_rs",
"puffin-interpreter",
]
@ -2708,12 +2710,6 @@ dependencies = [
"syn 2.0.38",
]
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "security-framework"
version = "2.9.2"

5
crates/gourgeist/Cargo.toml
View file

@ -14,8 +14,10 @@ authors = { workspace = true }
license = { workspace = true }
[dependencies]
install-wheel-rs = { path = "../install-wheel-rs", optional = true }
distribution-filename = { path = "../distribution-filename" }
install-wheel-rs = { path = "../install-wheel-rs", optional = true }
platform-host = { path = "../platform-host" }
puffin-interpreter = { path = "../puffin-interpreter" }
camino = { workspace = true }
clap = { workspace = true }
@ -24,7 +26,6 @@ dirs = { workspace = true }
fs-err = { workspace = true }
reqwest = { workspace = true, optional = true, features = ["blocking"] }
rayon = { workspace = true, optional = true }
seahash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }

38
crates/gourgeist/src/bare.rs
View file

@ -8,10 +8,9 @@ use fs_err as fs;
#[cfg(unix)]
use fs_err::os::unix::fs::symlink;
use fs_err::File;
use puffin_interpreter::InterpreterInfo;
use tracing::info;
use crate::interpreter::InterpreterInfo;
/// The bash activate scripts with the venv dependent paths patches out
const ACTIVATE_TEMPLATES: &[(&str, &str)] = &[
("activate", include_str!("activator/activate")),
@ -111,10 +110,17 @@ pub(crate) fn create_bare_venv(
#[cfg(unix)]
{
symlink(base_python, &venv_python)?;
symlink("python", bin_dir.join(format!("python{}", info.major)))?;
symlink(
"python",
bin_dir.join(format!("python{}.{}", info.major, info.minor)),
bin_dir.join(format!("python{}", info.simple_version().0)),
)?;
symlink(
"python",
bin_dir.join(format!(
"python{}.{}",
info.simple_version().0,
info.simple_version().1
)),
)?;
}
@ -124,7 +130,11 @@ pub(crate) fn create_bare_venv(
.replace("{{ VIRTUAL_ENV_DIR }}", location.as_str())
.replace(
"{{ RELATIVE_SITE_PACKAGES }}",
&format!("../lib/python{}.{}/site-packages", info.major, info.minor),
&format!(
"../lib/python{}.{}/site-packages",
info.simple_version().0,
info.simple_version().1
),
);
fs::write(bin_dir.join(name), activator)?;
}
@ -142,12 +152,18 @@ pub(crate) fn create_bare_venv(
let pyvenv_cfg_data = &[
("home", python_home),
("implementation", "CPython".to_string()),
("version_info", info.python_version.clone()),
("version_info", info.markers().python_version.string.clone()),
("gourgeist", env!("CARGO_PKG_VERSION").to_string()),
// I wouldn't allow this option anyway
("include-system-site-packages", "false".to_string()),
("base-prefix", info.base_prefix.clone()),
("base-exec-prefix", info.base_exec_prefix.clone()),
(
"base-prefix",
info.base_prefix().to_string_lossy().to_string(),
),
(
"base-exec-prefix",
info.base_exec_prefix().to_string_lossy().to_string(),
),
("base-executable", base_python.to_string()),
];
let mut pyvenv_cfg = BufWriter::new(File::create(location.join("pyvenv.cfg"))?);
@ -157,7 +173,11 @@ pub(crate) fn create_bare_venv(
// TODO: This is different on windows
let site_packages = location
.join("lib")
.join(format!("python{}.{}", info.major, info.minor))
.join(format!(
"python{}.{}",
info.simple_version().0,
info.simple_version().1
))
.join("site-packages");
fs::create_dir_all(&site_packages)?;
// Install _virtualenv.py patch.

159
crates/gourgeist/src/interpreter.rs
View file

@ -1,160 +1,5 @@
use std::io;
use std::io::{BufReader, Write};
use std::path::Path;
use std::process::{Command, Stdio};
use std::time::SystemTime;
use camino::{Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
use serde::{Deserialize, Serialize};
use tracing::{debug, error, warn};
use crate::{crate_cache_dir, Error};
const QUERY_PYTHON: &str = include_str!("query_python.py");
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct InterpreterInfo {
pub base_exec_prefix: String,
pub base_prefix: String,
pub major: u8,
pub minor: u8,
pub python_version: String,
}
/// Gets the interpreter.rs info, either cached or by running it.
pub fn get_interpreter_info(interpreter: impl AsRef<Path>) -> Result<InterpreterInfo, Error> {
let interpreter = Utf8Path::from_path(interpreter.as_ref())
.ok_or_else(|| Error::NonUTF8Path(interpreter.as_ref().to_path_buf()))?;
let cache_dir = crate_cache_dir()?.join("interpreter_info");
let index = seahash::hash(interpreter.as_str().as_bytes());
let cache_file = cache_dir.join(index.to_string()).with_extension("json");
let modified = fs::metadata(interpreter)?
.modified()?
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
if cache_file.exists() {
let cache_entry: Result<CacheEntry, String> = File::open(&cache_file)
.map_err(|err| err.to_string())
.and_then(|cache_reader| {
serde_json::from_reader(BufReader::new(cache_reader)).map_err(|err| err.to_string())
});
match cache_entry {
Ok(cache_entry) => {
debug!("Using cache entry {cache_file}");
if modified == cache_entry.modified && interpreter == cache_entry.interpreter {
return Ok(cache_entry.interpreter_info);
}
debug!(
"Removing mismatching cache entry {cache_file} ({} {} {} {})",
modified, cache_entry.modified, interpreter, cache_entry.interpreter
);
if let Err(remove_err) = fs::remove_file(&cache_file) {
warn!("Failed to mismatching cache file at {cache_file}: {remove_err}");
}
}
Err(cache_err) => {
debug!("Removing broken cache entry {cache_file} ({cache_err})");
if let Err(remove_err) = fs::remove_file(&cache_file) {
warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})");
}
}
}
}
let interpreter_info = query_interpreter(interpreter)?;
fs::create_dir_all(&cache_dir)?;
let cache_entry = CacheEntry {
interpreter: interpreter.to_path_buf(),
modified,
interpreter_info: interpreter_info.clone(),
};
let mut cache_writer = File::create(&cache_file)?;
serde_json::to_writer(&mut cache_writer, &cache_entry).map_err(io::Error::from)?;
Ok(interpreter_info)
}
#[derive(Clone, Debug, Deserialize, Serialize)]
struct CacheEntry {
interpreter: Utf8PathBuf,
modified: u128,
interpreter_info: InterpreterInfo,
}
/// Runs a python script that returns the relevant info about the interpreter.rs as json
fn query_interpreter(interpreter: &Utf8Path) -> Result<InterpreterInfo, Error> {
let mut child = Command::new(interpreter)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
if let Some(mut stdin) = child.stdin.take() {
stdin
.write_all(QUERY_PYTHON.as_bytes())
.map_err(|err| Error::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err,
})?;
}
let output = child.wait_with_output()?;
let stdout = String::from_utf8(output.stdout).unwrap_or_else(|err| {
// At this point, there was most likely an error caused by a non-utf8 character, so we're in
// an ugly case but still very much want to give the user a chance
error!(
"The stdout of the failed call of the call to {} contains non-utf8 characters",
interpreter
);
String::from_utf8_lossy(err.as_bytes()).to_string()
});
let stderr = String::from_utf8(output.stderr).unwrap_or_else(|err| {
error!(
"The stderr of the failed call of the call to {} contains non-utf8 characters",
interpreter
);
String::from_utf8_lossy(err.as_bytes()).to_string()
});
// stderr isn't technically a criterion for success, but i don't know of any cases where there
// should be stderr output and if there is, we want to know
if !output.status.success() || !stderr.trim().is_empty() {
return Err(Error::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err: io::Error::new(
io::ErrorKind::Other,
format!(
"Querying python at {} failed with status {}:\n--- stdout:\n{}\n--- stderr:\n{}",
interpreter,
output.status,
stdout.trim(),
stderr.trim()
),
)
});
}
let data = serde_json::from_str::<InterpreterInfo>(&stdout).map_err(|err|
Error::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err: io::Error::new(
io::ErrorKind::Other,
format!(
"Querying python at {} did not return the expected data ({}):\n--- stdout:\n{}\n--- stderr:\n{}",
interpreter,
err,
stdout.trim(),
stderr.trim()
)
)
}
)?;
Ok(data)
}
use camino::Utf8PathBuf;
use tracing::debug;
/// Parse the value of the `-p`/`--python` option, which can be e.g. `3.11`, `python3.11`,
/// `tools/bin/python3.11` or `/usr/bin/python3.11`.

49
crates/gourgeist/src/lib.rs
View file

@ -1,5 +1,4 @@
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use camino::{Utf8Path, Utf8PathBuf};
@ -7,7 +6,9 @@ use dirs::cache_dir;
use tempfile::PersistError;
use thiserror::Error;
pub use interpreter::{get_interpreter_info, parse_python_cli, InterpreterInfo};
pub use interpreter::parse_python_cli;
use platform_host::PlatformError;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use crate::bare::create_bare_venv;
@ -53,44 +54,8 @@ pub enum Error {
},
#[error("{0} is not a valid UTF-8 path")]
NonUTF8Path(PathBuf),
}
/// Provides the paths inside a venv
#[derive(Debug, Clone)]
pub struct Venv(Utf8PathBuf);
impl Deref for Venv {
type Target = Utf8Path;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Venv {
pub fn new(location: impl Into<PathBuf>) -> Result<Self, Error> {
let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?;
Ok(Self(location))
}
/// Returns the location of the python interpreter
pub fn python_interpreter(&self) -> PathBuf {
#[cfg(unix)]
{
self.0.join("bin").join("python").into_std_path_buf()
}
#[cfg(windows)]
{
self.0
.join("Scripts")
.join("python.exe")
.into_std_path_buf()
}
#[cfg(not(any(unix, windows)))]
{
compile_error!("Only windows and unix (linux, mac os, etc.) are supported")
}
}
#[error(transparent)]
Platform(#[from] PlatformError),
}
pub(crate) fn crate_cache_dir() -> io::Result<Utf8PathBuf> {
@ -106,7 +71,7 @@ pub fn create_venv(
base_python: impl AsRef<Path>,
info: &InterpreterInfo,
bare: bool,
) -> Result<Venv, Error> {
) -> Result<Virtualenv, Error> {
let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?;
let base_python = Utf8Path::from_path(base_python.as_ref())
.ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?;
@ -128,5 +93,5 @@ pub fn create_venv(
}
}
Ok(Venv(location))
Ok(Virtualenv::new_prefix(location.as_std_path(), info))
}

10
crates/gourgeist/src/main.rs
View file

@ -4,13 +4,14 @@ use std::time::Instant;
use camino::Utf8PathBuf;
use clap::Parser;
use gourgeist::{create_venv, parse_python_cli};
use platform_host::Platform;
use puffin_interpreter::InterpreterInfo;
use tracing::info;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter};
use gourgeist::{create_venv, get_interpreter_info, parse_python_cli};
#[derive(Parser, Debug)]
struct Cli {
path: Option<Utf8PathBuf>,
@ -24,8 +25,9 @@ fn run() -> Result<(), gourgeist::Error> {
let cli = Cli::parse();
let location = cli.path.unwrap_or(Utf8PathBuf::from(".venv"));
let python = parse_python_cli(cli.python)?;
let data = get_interpreter_info(&python)?;
create_venv(location, &python, &data, cli.bare)?;
let platform = Platform::current()?;
let info = InterpreterInfo::query_cached(python.as_std_path(), platform, None).unwrap();
create_venv(location, &python, &info, cli.bare)?;
Ok(())
}

4
crates/gourgeist/src/packages.rs
View file

@ -12,9 +12,9 @@ use tracing::debug;
use distribution_filename::WheelFilename;
use install_wheel_rs::{install_wheel, InstallLocation};
use puffin_interpreter::InterpreterInfo;
use crate::bare::VenvPaths;
use crate::interpreter::InterpreterInfo;
use crate::{crate_cache_dir, Error};
pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
@ -51,7 +51,7 @@ pub(crate) fn install_base_packages(
info: &InterpreterInfo,
paths: &VenvPaths,
) -> Result<(), Error> {
let install_location = InstallLocation::new(location.canonicalize()?, (info.major, info.minor));
let install_location = InstallLocation::new(location.canonicalize()?, info.simple_version());
let install_location = install_location.acquire_lock()?;
// TODO: Use the json api instead

18
crates/gourgeist/src/query_python.py
View file

@ -1,18 +0,0 @@
import json
import sys
from platform import python_version
def main():
data = {
"base_exec_prefix": sys.base_exec_prefix,
"base_prefix": sys.base_prefix,
"major": sys.version_info.major,
"minor": sys.version_info.minor,
"python_version": python_version(),
}
print(json.dumps(data))
if __name__ == "__main__":
main()

6
crates/gourgeist/venv_checker.py
View file

@ -16,11 +16,7 @@ def main():
output = check_output(["bash"], input=command, text=True).strip()
assert output.startswith("usage:"), output
output = (
check_output([venv_python, "imasnake.py"], text=True)
.strip()
.splitlines()
)
output = check_output([venv_python, "imasnake.py"], text=True).strip().splitlines()
assert output[0] == str(project_root.joinpath(venv_python)), output
assert not output[2].startswith(str(project_root)), output
assert output[3] == str(project_root.joinpath(venv_name)), output

18
crates/install-wheel-rs/src/install_location.rs
View file

@ -75,14 +75,14 @@ impl AsRef<Path> for LockedDir {
/// non-deterministically fail.
pub struct InstallLocation<T> {
/// absolute path
venv_base: T,
venv_root: T,
python_version: (u8, u8),
}
impl<T: AsRef<Path>> InstallLocation<T> {
pub fn new(venv_base: T, python_version: (u8, u8)) -> Self {
Self {
venv_base,
venv_root: venv_base,
python_version,
}
}
@ -90,10 +90,10 @@ impl<T: AsRef<Path>> InstallLocation<T> {
/// Returns the location of the `python` interpreter.
pub fn python(&self) -> PathBuf {
if cfg!(windows) {
self.venv_base.as_ref().join("Scripts").join("python.exe")
self.venv_root.as_ref().join("Scripts").join("python.exe")
} else {
// canonicalize on python would resolve the symlink
self.venv_base.as_ref().join("bin").join("python")
self.venv_root.as_ref().join("bin").join("python")
}
}
@ -101,25 +101,25 @@ impl<T: AsRef<Path>> InstallLocation<T> {
self.python_version
}
pub fn venv_base(&self) -> &T {
&self.venv_base
pub fn venv_root(&self) -> &T {
&self.venv_root
}
}
impl InstallLocation<PathBuf> {
pub fn acquire_lock(&self) -> io::Result<InstallLocation<LockedDir>> {
let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_base)? {
let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_root)? {
locked_dir
} else {
warn!(
"Could not acquire exclusive lock for installing, is another installation process \
running? Sleeping until lock becomes free"
);
LockedDir::acquire(&self.venv_base)?
LockedDir::acquire(&self.venv_root)?
};
Ok(InstallLocation {
venv_base: locked_dir,
venv_root: locked_dir,
python_version: self.python_version,
})
}

9
crates/install-wheel-rs/src/linker.rs
View file

@ -29,7 +29,7 @@ pub fn install_wheel(
wheel: impl AsRef<Path>,
link_mode: LinkMode,
) -> Result<(), Error> {
let base_location = location.venv_base();
let root = location.venv_root();
// TODO(charlie): Pass this in.
let site_packages_python = format!(
@ -38,10 +38,9 @@ pub fn install_wheel(
location.python_version().1
);
let site_packages = if cfg!(target_os = "windows") {
base_location.as_ref().join("Lib").join("site-packages")
root.as_ref().join("Lib").join("site-packages")
} else {
base_location
.as_ref()
root.as_ref()
.join("lib")
.join(site_packages_python)
.join("site-packages")
@ -88,7 +87,7 @@ pub fn install_wheel(
if data_dir.is_dir() {
debug!(name, "Installing data");
install_data(
base_location.as_ref(),
root.as_ref(),
&site_packages,
&data_dir,
&name,

8
crates/install-wheel-rs/src/wheel.rs
View file

@ -722,7 +722,7 @@ fn install_script(
/// Move the files from the .data directory to the right location in the venv
#[allow(clippy::too_many_arguments)]
pub(crate) fn install_data(
venv_base: &Path,
venv_root: &Path,
site_packages: &Path,
data_dir: &Path,
dist_name: &str,
@ -736,7 +736,7 @@ pub(crate) fn install_data(
match data_entry.file_name().as_os_str().to_str() {
Some("data") => {
// Move the content of the folder to the root of the venv
move_folder_recorded(&data_entry.path(), venv_base, site_packages, record)?;
move_folder_recorded(&data_entry.path(), venv_root, site_packages, record)?;
}
Some("scripts") => {
for file in fs::read_dir(data_entry.path())? {
@ -762,7 +762,7 @@ pub(crate) fn install_data(
}
}
Some("headers") => {
let target_path = venv_base
let target_path = venv_root
.join("include")
.join("site")
.join(format!(
@ -901,7 +901,7 @@ pub fn install_wheel(
let name = &filename.distribution;
let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str());
let base_location = location.venv_base();
let base_location = location.venv_root();
let site_packages_python = format!(
"python{}.{}",

2
crates/pep440-rs/python/pep440_rs/__init__.pyi
View file

@ -1,7 +1,6 @@
# Generated by `stubgen -p pep440_rs`
from typing import Any, ClassVar
class Version:
dev: Any
epoch: Any
@ -24,7 +23,6 @@ class Version:
def __lt__(self, other) -> Any: ...
def __ne__(self, other) -> Any: ...
class VersionSpecifier:
__hash__: ClassVar[None] = ...

21
crates/puffin-build-cli/src/main.rs
View file

@ -20,7 +20,7 @@ use platform_host::Platform;
use puffin_build::SourceDistributionBuilder;
use puffin_client::RegistryClientBuilder;
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
#[derive(Parser)]
struct Args {
@ -44,17 +44,22 @@ async fn run() -> Result<()> {
};
let dirs = ProjectDirs::from("", "", "puffin");
let cache = dirs.as_ref().map(ProjectDirs::cache_dir);
let cache = dirs
.as_ref()
.map(|dir| ProjectDirs::cache_dir(dir).to_path_buf());
let platform = Platform::current()?;
let python = PythonExecutable::from_env(platform, cache)?;
let venv = Virtualenv::from_env(platform, cache.as_deref())?;
let interpreter_info = gourgeist::get_interpreter_info(python.executable())?;
let build_dispatch =
BuildDispatch::new(RegistryClientBuilder::default().build(), python, cache);
let build_dispatch = BuildDispatch::new(
RegistryClientBuilder::default().build(),
cache,
venv.interpreter_info().clone(),
fs::canonicalize(venv.python_executable())?,
);
let builder =
SourceDistributionBuilder::setup(&args.sdist, &interpreter_info, &build_dispatch).await?;
SourceDistributionBuilder::setup(&args.sdist, venv.interpreter_info(), &build_dispatch)
.await?;
let wheel = builder.build(&wheel_dir)?;
println!("Wheel built to {}", wheel_dir.join(wheel).display());
Ok(())

1
crates/puffin-build/Cargo.toml
View file

@ -15,6 +15,7 @@ gourgeist = { path = "../gourgeist" }
pep508_rs = { path = "../pep508-rs" }
platform-host = { path = "../platform-host" }
platform-tags = { path = "../platform-tags" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-traits = { path = "../puffin-traits" }
anyhow = { workspace = true }

25
crates/puffin-build/src/lib.rs
View file

@ -19,8 +19,8 @@ use thiserror::Error;
use tracing::{debug, instrument};
use zip::ZipArchive;
use gourgeist::{InterpreterInfo, Venv};
use pep508_rs::Requirement;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_traits::BuildContext;
#[derive(Error, Debug)]
@ -88,7 +88,7 @@ pub struct SourceDistributionBuilder {
source_tree: PathBuf,
/// `Some` if this is a PEP 517 build
pep517_backend: Option<Pep517Backend>,
venv: Venv,
venv: Virtualenv,
/// Populated if `prepare_metadata_for_build_wheel` was called.
///
/// > If the build frontend has previously called prepare_metadata_for_build_wheel and depends
@ -154,7 +154,7 @@ impl SourceDistributionBuilder {
}
let venv = gourgeist::create_venv(
temp_dir.path().join("venv"),
build_context.python().executable(),
build_context.base_python(),
interpreter_info,
true,
)?;
@ -211,8 +211,7 @@ impl SourceDistributionBuilder {
print()
"#, pep517_backend.backend_import(), escape_path_for_python(&metadata_directory)
};
let output =
run_python_script(&self.venv.python_interpreter(), &script, &self.source_tree)?;
let output = run_python_script(&self.venv.python_executable(), &script, &self.source_tree)?;
if !output.status.success() {
return Err(Error::from_command_output(
"Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(),
@ -259,7 +258,7 @@ impl SourceDistributionBuilder {
self.pep517_build_wheel(&wheel_dir, pep517_backend)
} else {
// We checked earlier that setup.py exists
let python_interpreter = self.venv.python_interpreter();
let python_interpreter = self.venv.python_executable();
let output = Command::new(&python_interpreter)
.args(["setup.py", "bdist_wheel"])
.current_dir(&self.source_tree)
@ -310,8 +309,7 @@ impl SourceDistributionBuilder {
print(backend.build_wheel("{}", metadata_directory={}))
"#, pep517_backend.backend_import(), escaped_wheel_dir, metadata_directory
};
let output =
run_python_script(&self.venv.python_interpreter(), &script, &self.source_tree)?;
let output = run_python_script(&self.venv.python_executable(), &script, &self.source_tree)?;
if !output.status.success() {
return Err(Error::from_command_output(
"Build backend failed to build wheel through `build_wheel()` ".to_string(),
@ -346,13 +344,8 @@ async fn create_pep517_build_environment(
data: &InterpreterInfo,
pep517_backend: &Pep517Backend,
build_context: &impl BuildContext,
) -> Result<Venv, Error> {
let venv = gourgeist::create_venv(
root.join(".venv"),
build_context.python().executable(),
data,
true,
)?;
) -> Result<Virtualenv, Error> {
let venv = gourgeist::create_venv(root.join(".venv"), build_context.base_python(), data, true)?;
let resolved_requirements = build_context
.resolve(&pep517_backend.requirements)
.await
@ -377,7 +370,7 @@ async fn create_pep517_build_environment(
print(json.dumps(requires))
"#, pep517_backend.backend_import()
};
let output = run_python_script(&venv.python_interpreter(), &script, source_tree)?;
let output = run_python_script(&venv.python_executable(), &script, source_tree)?;
if !output.status.success() {
return Err(Error::from_command_output(
"Build backend failed to determine extras requires with `get_requires_for_build_wheel`"

6
crates/puffin-cli/src/commands/freeze.rs
View file

@ -5,7 +5,7 @@ use tracing::debug;
use platform_host::Platform;
use puffin_installer::SitePackages;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use crate::commands::ExitStatus;
use crate::printer::Printer;
@ -14,10 +14,10 @@ use crate::printer::Printer;
pub(crate) fn freeze(cache: Option<&Path>, _printer: Printer) -> Result<ExitStatus> {
// Detect the current Python interpreter.
let platform = Platform::current()?;
let python = PythonExecutable::from_env(platform, cache)?;
let python = Virtualenv::from_env(platform, cache)?;
debug!(
"Using Python interpreter: {}",
python.executable().display()
python.python_executable().display()
);
// Build the installed index.

22
crates/puffin-cli/src/commands/pip_compile.rs
View file

@ -1,7 +1,7 @@
use std::env;
use std::fmt::Write;
use std::io::{stdout, BufWriter};
use std::path::Path;
use std::{env, fs};
use anyhow::Result;
use colored::Colorize;
@ -13,7 +13,7 @@ use platform_tags::Tags;
use pubgrub::report::Reporter;
use puffin_client::RegistryClientBuilder;
use puffin_dispatch::BuildDispatch;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use puffin_resolver::ResolutionMode;
use tracing::debug;
@ -50,16 +50,19 @@ pub(crate) async fn pip_compile(
// Detect the current Python interpreter.
let platform = Platform::current()?;
let python = PythonExecutable::from_env(platform, cache)?;
let venv = Virtualenv::from_env(platform, cache)?;
debug!(
"Using Python {} at {}",
python.markers().python_version,
python.executable().display()
venv.interpreter_info().markers().python_version,
venv.python_executable().display()
);
// Determine the compatible platform tags.
let tags = Tags::from_env(python.platform(), python.simple_version())?;
let tags = Tags::from_env(
venv.interpreter_info().platform(),
venv.interpreter_info().simple_version(),
)?;
// Instantiate a client.
let client = {
@ -78,8 +81,9 @@ pub(crate) async fn pip_compile(
let build_dispatch = BuildDispatch::new(
RegistryClientBuilder::default().build(),
python.clone(),
cache,
cache.map(Path::to_path_buf),
venv.interpreter_info().clone(),
fs::canonicalize(venv.python_executable())?,
);
// Resolve the dependencies.
@ -87,7 +91,7 @@ pub(crate) async fn pip_compile(
requirements,
constraints,
mode,
python.markers(),
venv.interpreter_info().markers(),
&tags,
&client,
&build_dispatch,

15
crates/puffin-cli/src/commands/pip_sync.rs
View file

@ -12,7 +12,7 @@ use platform_host::Platform;
use platform_tags::Tags;
use puffin_client::RegistryClientBuilder;
use puffin_installer::{Distribution, PartitionedRequirements, RemoteDistribution};
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use crate::commands::reporters::{
DownloadReporter, InstallReporter, UnzipReporter, WheelFinderReporter,
@ -58,10 +58,10 @@ pub(crate) async fn sync_requirements(
// Detect the current Python interpreter.
let platform = Platform::current()?;
let python = PythonExecutable::from_env(platform, cache)?;
let venv = Virtualenv::from_env(platform, cache)?;
debug!(
"Using Python interpreter: {}",
python.executable().display()
venv.python_executable().display()
);
// Partition into those that should be linked from the cache (`local`), those that need to be
@ -70,7 +70,7 @@ pub(crate) async fn sync_requirements(
local,
remote,
extraneous,
} = PartitionedRequirements::try_from_requirements(requirements, cache, &python)?;
} = PartitionedRequirements::try_from_requirements(requirements, cache, &venv)?;
// Nothing to do.
if remote.is_empty() && local.is_empty() && extraneous.is_empty() {
@ -90,7 +90,10 @@ pub(crate) async fn sync_requirements(
}
// Determine the current environment markers.
let tags = Tags::from_env(python.platform(), python.simple_version())?;
let tags = Tags::from_env(
venv.interpreter_info().platform(),
venv.interpreter_info().simple_version(),
)?;
// Instantiate a client.
let client = {
@ -226,7 +229,7 @@ pub(crate) async fn sync_requirements(
let wheels = unzips.into_iter().chain(local).collect::<Vec<_>>();
if !wheels.is_empty() {
let start = std::time::Instant::now();
puffin_installer::Installer::new(&python)
puffin_installer::Installer::new(&venv)
.with_link_mode(link_mode)
.with_reporter(InstallReporter::from(printer).with_length(wheels.len() as u64))
.install(&wheels)?;

8
crates/puffin-cli/src/commands/pip_uninstall.rs
View file

@ -8,7 +8,7 @@ use tracing::debug;
use pep508_rs::Requirement;
use platform_host::Platform;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use crate::commands::{elapsed, ExitStatus};
@ -32,14 +32,14 @@ pub(crate) async fn pip_uninstall(
// Detect the current Python interpreter.
let platform = Platform::current()?;
let python = PythonExecutable::from_env(platform, cache)?;
let venv = Virtualenv::from_env(platform, cache)?;
debug!(
"Using Python interpreter: {}",
python.executable().display()
venv.python_executable().display()
);
// Index the current `site-packages` directory.
let site_packages = puffin_installer::SitePackages::try_from_executable(&python)?;
let site_packages = puffin_installer::SitePackages::try_from_executable(&venv)?;
// Sort and deduplicate the requirements.
let packages = {

10
crates/puffin-cli/src/commands/venv.rs
View file

@ -5,6 +5,8 @@ use anyhow::Result;
use colored::Colorize;
use fs_err as fs;
use miette::{Diagnostic, IntoDiagnostic};
use platform_host::Platform;
use puffin_interpreter::InterpreterInfo;
use thiserror::Error;
use crate::commands::ExitStatus;
@ -37,7 +39,7 @@ enum VenvError {
#[error("Failed to extract Python interpreter info")]
#[diagnostic(code(puffin::venv::interpreter))]
InterpreterError(#[source] gourgeist::Error),
InterpreterError(#[source] anyhow::Error),
#[error("Failed to create virtual environment")]
#[diagnostic(code(puffin::venv::creation))]
@ -59,8 +61,10 @@ fn venv_impl(
.or_else(|_| which::which("python"))
.map_err(|_| VenvError::PythonNotFound)?
};
let interpreter_info =
gourgeist::get_interpreter_info(&base_python).map_err(VenvError::InterpreterError)?;
let platform = Platform::current().into_diagnostic()?;
// TODO(konstin): Add caching
let interpreter_info = InterpreterInfo::query_cached(&base_python, platform, None)
.map_err(VenvError::InterpreterError)?;
writeln!(
printer,

58
crates/puffin-dispatch/src/lib.rs
View file

@ -10,7 +10,6 @@ use anyhow::Context;
use itertools::Itertools;
use tempfile::tempdir;
use gourgeist::Venv;
use pep508_rs::Requirement;
use platform_tags::Tags;
use puffin_build::SourceDistributionBuilder;
@ -18,7 +17,7 @@ use puffin_client::RegistryClient;
use puffin_installer::{
uninstall, Downloader, Installer, PartitionedRequirements, RemoteDistribution, Unzipper,
};
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_resolver::{ResolutionMode, Resolver, WheelFinder};
use puffin_traits::BuildContext;
use tracing::debug;
@ -27,19 +26,23 @@ use tracing::debug;
/// documentation.
pub struct BuildDispatch {
client: RegistryClient,
python: PythonExecutable,
cache: Option<PathBuf>,
interpreter_info: InterpreterInfo,
base_python: PathBuf,
}
impl BuildDispatch {
pub fn new<T>(client: RegistryClient, python: PythonExecutable, cache: Option<T>) -> Self
where
T: Into<PathBuf>,
{
pub fn new(
client: RegistryClient,
cache: Option<PathBuf>,
interpreter_info: InterpreterInfo,
base_python: PathBuf,
) -> Self {
Self {
client,
python,
cache: cache.map(Into::into),
cache,
interpreter_info,
base_python,
}
}
}
@ -49,8 +52,12 @@ impl BuildContext for BuildDispatch {
self.cache.as_deref()
}
fn python(&self) -> &PythonExecutable {
&self.python
fn interpreter_info(&self) -> &InterpreterInfo {
&self.interpreter_info
}
fn base_python(&self) -> &Path {
&self.base_python
}
fn resolve<'a>(
@ -58,12 +65,15 @@ impl BuildContext for BuildDispatch {
requirements: &'a [Requirement],
) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<Requirement>>> + 'a>> {
Box::pin(async {
let tags = Tags::from_env(self.python.platform(), self.python.simple_version())?;
let tags = Tags::from_env(
self.interpreter_info.platform(),
self.interpreter_info.simple_version(),
)?;
let resolver = Resolver::new(
requirements.to_vec(),
Vec::default(),
ResolutionMode::Highest,
self.python.markers(),
self.interpreter_info.markers(),
&tags,
&self.client,
self,
@ -78,25 +88,20 @@ impl BuildContext for BuildDispatch {
fn install<'a>(
&'a self,
requirements: &'a [Requirement],
venv: &'a Venv,
venv: &'a Virtualenv,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + 'a>> {
Box::pin(async move {
debug!(
"Install in {} requirements {}",
venv.as_str(),
venv.root().display(),
requirements.iter().map(ToString::to_string).join(", ")
);
let python = self.python().with_venv(venv.as_std_path());
let PartitionedRequirements {
local,
remote,
extraneous,
} = PartitionedRequirements::try_from_requirements(
requirements,
self.cache(),
&python,
)?;
} = PartitionedRequirements::try_from_requirements(requirements, self.cache(), venv)?;
if !extraneous.is_empty() {
debug!(
@ -117,7 +122,10 @@ impl BuildContext for BuildDispatch {
remote.iter().map(ToString::to_string).join(", ")
);
let tags = Tags::from_env(python.platform(), python.simple_version())?;
let tags = Tags::from_env(
self.interpreter_info.platform(),
self.interpreter_info.simple_version(),
)?;
let resolution = WheelFinder::new(&tags, &self.client)
.resolve(&remote)
.await?;
@ -144,7 +152,7 @@ impl BuildContext for BuildDispatch {
.join(", ")
);
let wheels = unzips.into_iter().chain(local).collect::<Vec<_>>();
Installer::new(&python).install(&wheels)?;
Installer::new(venv).install(&wheels)?;
Ok(())
})
}
@ -155,8 +163,8 @@ impl BuildContext for BuildDispatch {
wheel_dir: &'a Path,
) -> Pin<Box<dyn Future<Output = anyhow::Result<String>> + 'a>> {
Box::pin(async move {
let interpreter_info = gourgeist::get_interpreter_info(self.python.executable())?;
let builder = SourceDistributionBuilder::setup(sdist, &interpreter_info, self).await?;
let builder =
SourceDistributionBuilder::setup(sdist, &self.interpreter_info, self).await?;
Ok(builder.build(wheel_dir)?)
})
}

12
crates/puffin-installer/src/installer.rs
View file

@ -2,22 +2,22 @@ use anyhow::{Error, Result};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use pep440_rs::Version;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use crate::CachedDistribution;
pub struct Installer<'a> {
python: &'a PythonExecutable,
venv: &'a Virtualenv,
link_mode: install_wheel_rs::linker::LinkMode,
reporter: Option<Box<dyn Reporter>>,
}
impl<'a> Installer<'a> {
/// Initialize a new installer.
pub fn new(python: &'a PythonExecutable) -> Self {
pub fn new(venv: &'a Virtualenv) -> Self {
Self {
python,
venv,
link_mode: install_wheel_rs::linker::LinkMode::default(),
reporter: None,
}
@ -43,8 +43,8 @@ impl<'a> Installer<'a> {
tokio::task::block_in_place(|| {
wheels.par_iter().try_for_each(|wheel| {
let location = install_wheel_rs::InstallLocation::new(
self.python.venv().to_path_buf(),
self.python.simple_version(),
self.venv.root(),
self.venv.interpreter_info().simple_version(),
);
install_wheel_rs::linker::install_wheel(&location, wheel.path(), self.link_mode)?;

6
crates/puffin-installer/src/plan.rs
View file

@ -4,7 +4,7 @@ use anyhow::Result;
use tracing::debug;
use pep508_rs::Requirement;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use crate::{CachedDistribution, InstalledDistribution, LocalIndex, SitePackages};
@ -30,10 +30,10 @@ impl PartitionedRequirements {
pub fn try_from_requirements(
requirements: &[Requirement],
cache: Option<&Path>,
python: &PythonExecutable,
venv: &Virtualenv,
) -> Result<Self> {
// Index all the already-installed packages in site-packages.
let mut site_packages = SitePackages::try_from_executable(python)?;
let mut site_packages = SitePackages::try_from_executable(venv)?;
// Index all the already-downloaded wheels in the cache.
let local_index = if let Some(cache) = cache {

6
crates/puffin-installer/src/site_packages.rs
View file

@ -2,8 +2,8 @@ use std::collections::BTreeMap;
use anyhow::Result;
use fs_err as fs;
use puffin_interpreter::Virtualenv;
use puffin_interpreter::PythonExecutable;
use puffin_package::package_name::PackageName;
use crate::InstalledDistribution;
@ -13,10 +13,10 @@ pub struct SitePackages(BTreeMap<PackageName, InstalledDistribution>);
impl SitePackages {
/// Build an index of installed packages from the given Python executable.
pub fn try_from_executable(python: &PythonExecutable) -> Result<Self> {
pub fn try_from_executable(venv: &Virtualenv) -> Result<Self> {
let mut index = BTreeMap::new();
for entry in fs::read_dir(python.site_packages())? {
for entry in fs::read_dir(venv.site_packages())? {
let entry = entry?;
if entry.file_type()?.is_dir() {
if let Some(dist_info) = InstalledDistribution::try_from_path(&entry.path())? {

4
crates/puffin-interpreter/Cargo.toml
View file

@ -11,12 +11,14 @@ license = { workspace = true }
[dependencies]
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
pep508_rs = { path = "../pep508-rs", features = ["serde"] }
platform-host = { path = "../platform-host" }
anyhow = { workspace = true }
cacache = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
serde = { workspace = true, features = ["derive"] }

39
crates/puffin-interpreter/src/get_interpreter_info.py Normal file
View file

@ -0,0 +1,39 @@
import json
import os
import platform
import sys
def format_full_version(info):
version = "{0.major}.{0.minor}.{0.micro}".format(info)
kind = info.releaselevel
if kind != "final":
version += kind[0] + str(info.serial)
return version
if hasattr(sys, "implementation"):
implementation_version = format_full_version(sys.implementation.version)
implementation_name = sys.implementation.name
else:
implementation_version = "0"
implementation_name = ""
markers = {
"implementation_name": implementation_name,
"implementation_version": implementation_version,
"os_name": os.name,
"platform_machine": platform.machine(),
"platform_python_implementation": platform.python_implementation(),
"platform_release": platform.release(),
"platform_system": platform.system(),
"platform_version": platform.version(),
"python_full_version": platform.python_version(),
"python_version": ".".join(platform.python_version_tuple()[:2]),
"sys_platform": sys.platform,
}
interpreter_info = {
"markers": markers,
"base_prefix": sys.base_prefix,
"base_exec_prefix": sys.base_exec_prefix,
}
print(json.dumps(interpreter_info))

189
crates/puffin-interpreter/src/interpreter_info.rs Normal file
View file

@ -0,0 +1,189 @@
use std::io;
use std::path::{Path, PathBuf};
use std::process::Command;
use pep440_rs::Version;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::debug;
use crate::python_platform::PythonPlatform;
use pep508_rs::MarkerEnvironment;
use platform_host::Platform;
/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
pub struct InterpreterInfo {
pub(crate) platform: PythonPlatform,
pub(crate) markers: MarkerEnvironment,
pub(crate) base_exec_prefix: PathBuf,
pub(crate) base_prefix: PathBuf,
}
impl InterpreterInfo {
pub fn query_cached(
executable: &Path,
platform: Platform,
cache: Option<&Path>,
) -> anyhow::Result<Self> {
let info = InterpreterQueryResult::query_cached(executable, cache)?;
debug_assert!(
info.base_prefix == info.base_exec_prefix,
"Not a venv python: {}, prefix: {}",
executable.display(),
info.base_prefix.display()
);
Ok(Self {
platform: PythonPlatform(platform),
markers: info.markers,
base_exec_prefix: info.base_exec_prefix,
base_prefix: info.base_prefix,
})
}
}
impl InterpreterInfo {
/// Returns the path to the Python virtual environment.
pub fn platform(&self) -> &Platform {
&self.platform
}
/// Returns the [`MarkerEnvironment`] for this Python executable.
pub fn markers(&self) -> &MarkerEnvironment {
&self.markers
}
/// Returns the Python version.
pub fn version(&self) -> &Version {
&self.markers.python_version.version
}
/// Returns the Python version as a simple tuple.
pub fn simple_version(&self) -> (u8, u8) {
(
u8::try_from(self.version().release[0]).expect("invalid major version"),
u8::try_from(self.version().release[1]).expect("invalid minor version"),
)
}
pub fn base_exec_prefix(&self) -> &Path {
&self.base_exec_prefix
}
pub fn base_prefix(&self) -> &Path {
&self.base_prefix
}
}
#[derive(Debug, Error)]
pub(crate) enum InterpreterQueryError {
#[error(transparent)]
IO(#[from] io::Error),
#[error("Failed to query python interpreter at {interpreter}")]
PythonSubcommand {
interpreter: PathBuf,
#[source]
err: io::Error,
},
}
#[derive(Deserialize, Serialize)]
pub(crate) struct InterpreterQueryResult {
pub(crate) markers: MarkerEnvironment,
pub(crate) base_exec_prefix: PathBuf,
pub(crate) base_prefix: PathBuf,
}
impl InterpreterQueryResult {
/// Return the resolved [`InterpreterQueryResult`] for the given Python executable.
pub(crate) fn query(interpreter: &Path) -> Result<Self, InterpreterQueryError> {
let output = Command::new(interpreter)
.args(["-c", include_str!("get_interpreter_info.py")])
.output()
.map_err(|err| InterpreterQueryError::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err,
})?;
// stderr isn't technically a criterion for success, but i don't know of any cases where there
// should be stderr output and if there is, we want to know
if !output.status.success() || !output.stderr.is_empty() {
return Err(InterpreterQueryError::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err: io::Error::new(
io::ErrorKind::Other,
format!(
"Querying python at {} failed with status {}:\n--- stdout:\n{}\n--- stderr:\n{}",
interpreter.display(),
output.status,
String::from_utf8_lossy(&output.stdout).trim(),
String::from_utf8_lossy(&output.stderr).trim()
),
)
});
}
let data = serde_json::from_slice::<Self>(&output.stdout).map_err(|err|
InterpreterQueryError::PythonSubcommand {
interpreter: interpreter.to_path_buf(),
err: io::Error::new(
io::ErrorKind::Other,
format!(
"Querying python at {} did not return the expected data ({}):\n--- stdout:\n{}\n--- stderr:\n{}",
interpreter.display(),
err,
String::from_utf8_lossy(&output.stdout).trim(),
String::from_utf8_lossy(&output.stderr).trim()
)
)
}
)?;
Ok(data)
}
/// A wrapper around [`markers::query_interpreter_info`] to cache the computed markers.
///
/// Running a Python script is (relatively) expensive, and the markers won't change
/// unless the Python executable changes, so we use the executable's last modified
/// time as a cache key.
pub(crate) fn query_cached(executable: &Path, cache: Option<&Path>) -> anyhow::Result<Self> {
// Read from the cache.
let key = if let Some(cache) = cache {
if let Ok(key) = cache_key(executable) {
if let Ok(data) = cacache::read_sync(cache, &key) {
debug!("Using cached markers for {}", executable.display());
return Ok(serde_json::from_slice::<Self>(&data)?);
}
Some(key)
} else {
None
}
} else {
None
};
// Otherwise, run the Python script.
debug!("Detecting markers for {}", executable.display());
let info = Self::query(executable)?;
// Write to the cache.
if let Some(cache) = cache {
if let Some(key) = key {
cacache::write_sync(cache, key, serde_json::to_vec(&info)?)?;
}
}
Ok(info)
}
}
/// Create a cache key for the Python executable, consisting of the executable's
/// last modified time and the executable's path.
fn cache_key(executable: &Path) -> anyhow::Result<String> {
let modified = executable
.metadata()?
.modified()?
.duration_since(std::time::UNIX_EPOCH)?
.as_millis();
Ok(format!("puffin:v0:{}:{}", executable.display(), modified))
}

103
crates/puffin-interpreter/src/lib.rs
View file

@ -1,103 +1,6 @@
use std::path::{Path, PathBuf};
pub use crate::interpreter_info::InterpreterInfo;
pub use crate::virtual_env::Virtualenv;
use anyhow::Result;
use pep440_rs::Version;
use pep508_rs::MarkerEnvironment;
use platform_host::Platform;
use crate::python_platform::PythonPlatform;
mod markers;
mod interpreter_info;
mod python_platform;
mod virtual_env;
/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
pub struct PythonExecutable {
platform: PythonPlatform,
venv: PathBuf,
executable: PathBuf,
markers: MarkerEnvironment,
}
impl PythonExecutable {
/// Detect the current Python executable from the host environment.
pub fn from_env(platform: Platform, cache: Option<&Path>) -> Result<Self> {
let platform = PythonPlatform::from(platform);
let venv = virtual_env::detect_virtual_env(&platform)?;
let executable = platform.venv_python(&venv);
let markers = markers::detect_cached_markers(&executable, cache)?;
Ok(Self {
platform,
venv,
executable,
markers,
})
}
pub fn from_venv(platform: Platform, venv: &Path, cache: Option<&Path>) -> Result<Self> {
let platform = PythonPlatform::from(platform);
let executable = platform.venv_python(venv);
let markers = markers::detect_cached_markers(&executable, cache)?;
Ok(Self {
platform,
venv: venv.to_path_buf(),
executable,
markers,
})
}
/// Create a [`PythonExecutable`] for a venv with a known base [`PythonExecutable`].
#[must_use]
pub fn with_venv(&self, venv: &Path) -> Self {
let executable = self.platform.venv_python(venv);
Self {
venv: venv.to_path_buf(),
executable,
..self.clone()
}
}
/// Returns the path to the Python virtual environment.
pub fn platform(&self) -> &Platform {
&self.platform
}
/// Returns the path to the `site-packages` directory inside a virtual environment.
pub fn site_packages(&self) -> PathBuf {
self.platform
.venv_site_packages(self.venv(), self.simple_version())
}
/// Returns the path to the Python virtual environment.
pub fn venv(&self) -> &Path {
self.venv.as_path()
}
/// Returns the path to the Python executable.
pub fn executable(&self) -> &Path {
self.executable.as_path()
}
/// Returns the [`MarkerEnvironment`] for this Python executable.
pub fn markers(&self) -> &MarkerEnvironment {
&self.markers
}
/// Returns the Python version.
pub fn version(&self) -> &Version {
&self.markers.python_version.version
}
/// Returns the Python version as a simple tuple.
pub fn simple_version(&self) -> (u8, u8) {
(
u8::try_from(self.version().release[0]).expect("invalid major version"),
u8::try_from(self.version().release[1]).expect("invalid minor version"),
)
}
}

110
crates/puffin-interpreter/src/markers.rs
View file

@ -1,110 +0,0 @@
use std::ffi::OsStr;
use std::path::Path;
use std::process::{Command, Output};
use anyhow::{Context, Result};
use tracing::debug;
use pep508_rs::MarkerEnvironment;
/// Return the resolved [`MarkerEnvironment`] for the given Python executable.
pub(crate) fn detect_markers(python: impl AsRef<Path>) -> Result<MarkerEnvironment> {
let output = call_python(python.as_ref(), ["-c", CAPTURE_MARKERS_SCRIPT])?;
Ok(serde_json::from_slice::<MarkerEnvironment>(&output.stdout)?)
}
/// A wrapper around [`markers::detect_markers`] to cache the computed markers.
///
/// Running a Python script is (relatively) expensive, and the markers won't change
/// unless the Python executable changes, so we use the executable's last modified
/// time as a cache key.
pub(crate) fn detect_cached_markers(
executable: &Path,
cache: Option<&Path>,
) -> Result<MarkerEnvironment> {
// Read from the cache.
let key = if let Some(cache) = cache {
if let Ok(key) = cache_key(executable) {
if let Ok(data) = cacache::read_sync(cache, &key) {
debug!("Using cached markers for {}", executable.display());
return Ok(serde_json::from_slice::<MarkerEnvironment>(&data)?);
}
Some(key)
} else {
None
}
} else {
None
};
// Otherwise, run the Python script.
debug!("Detecting markers for {}", executable.display());
let markers = detect_markers(executable)?;
// Write to the cache.
if let Some(cache) = cache {
if let Some(key) = key {
cacache::write_sync(cache, key, serde_json::to_vec(&markers)?)?;
}
}
Ok(markers)
}
/// Create a cache key for the Python executable, consisting of the executable's
/// last modified time and the executable's path.
fn cache_key(executable: &Path) -> Result<String> {
let modified = executable
.metadata()?
.modified()?
.duration_since(std::time::UNIX_EPOCH)?
.as_millis();
Ok(format!("puffin:v0:{}:{}", executable.display(), modified))
}
const CAPTURE_MARKERS_SCRIPT: &str = "
import os
import sys
import platform
import json
def format_full_version(info):
version = '{0.major}.{0.minor}.{0.micro}'.format(info)
kind = info.releaselevel
if kind != 'final':
version += kind[0] + str(info.serial)
return version
if hasattr(sys, 'implementation'):
implementation_version = format_full_version(sys.implementation.version)
implementation_name = sys.implementation.name
else:
implementation_version = '0'
implementation_name = ''
bindings = {
'implementation_name': implementation_name,
'implementation_version': implementation_version,
'os_name': os.name,
'platform_machine': platform.machine(),
'platform_python_implementation': platform.python_implementation(),
'platform_release': platform.release(),
'platform_system': platform.system(),
'platform_version': platform.version(),
'python_full_version': platform.python_version(),
'python_version': '.'.join(platform.python_version_tuple()[:2]),
'sys_platform': sys.platform,
}
json.dump(bindings, sys.stdout)
sys.stdout.flush()
";
/// Run a Python script and return its output.
fn call_python<I, S>(python: &Path, args: I) -> Result<Output>
where
I: IntoIterator<Item = S>,
S: AsRef<OsStr>,
{
Command::new(python)
.args(args)
.output()
.context(format!("Failed to run `python` at: {:?}", &python))
}

14
crates/puffin-interpreter/src/python_platform.rs
View file

@ -6,22 +6,22 @@ use platform_host::{Os, Platform};
/// A Python-aware wrapper around [`Platform`].
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) struct PythonPlatform(Platform);
pub(crate) struct PythonPlatform(pub(crate) Platform);
impl PythonPlatform {
/// Returns the path to the `python` executable inside a virtual environment.
pub(crate) fn venv_python(&self, venv_base: impl AsRef<Path>) -> PathBuf {
pub(crate) fn venv_python(&self, venv_root: impl AsRef<Path>) -> PathBuf {
let python = if matches!(self.0.os(), Os::Windows) {
"python.exe"
} else {
"python"
};
self.venv_bin_dir(venv_base).join(python)
self.venv_bin_dir(venv_root).join(python)
}
/// Returns the directory in which the binaries are stored inside a virtual environment.
pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef<Path>) -> PathBuf {
let venv = venv_base.as_ref();
pub(crate) fn venv_bin_dir(&self, venv_root: impl AsRef<Path>) -> PathBuf {
let venv = venv_root.as_ref();
if matches!(self.0.os(), Os::Windows) {
let bin_dir = venv.join("Scripts");
if bin_dir.join("python.exe").exists() {
@ -43,10 +43,10 @@ impl PythonPlatform {
/// Returns the path to the `site-packages` directory inside a virtual environment.
pub(crate) fn venv_site_packages(
&self,
venv_base: impl AsRef<Path>,
venv_root: impl AsRef<Path>,
version: (u8, u8),
) -> PathBuf {
let venv = venv_base.as_ref();
let venv = venv_root.as_ref();
if matches!(self.0.os(), Os::Windows) {
venv.join("Lib").join("site-packages")
} else {

82
crates/puffin-interpreter/src/virtual_env.rs
View file

@ -1,11 +1,91 @@
use std::env;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use crate::InterpreterInfo;
use anyhow::{bail, Result};
use platform_host::Platform;
use tracing::debug;
use crate::python_platform::PythonPlatform;
/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
pub struct Virtualenv {
root: PathBuf,
interpreter_info: InterpreterInfo,
}
impl Virtualenv {
/// Venv the current Python executable from the host environment.
pub fn from_env(platform: Platform, cache: Option<&Path>) -> Result<Self> {
let platform = PythonPlatform::from(platform);
let venv = detect_virtual_env(&platform)?;
let executable = platform.venv_python(&venv);
let interpreter_info = InterpreterInfo::query_cached(&executable, platform.0, cache)?;
Ok(Self {
root: venv,
interpreter_info,
})
}
pub fn from_virtualenv(platform: Platform, root: &Path, cache: Option<&Path>) -> Result<Self> {
let platform = PythonPlatform::from(platform);
let executable = platform.venv_python(root);
let interpreter_info = InterpreterInfo::query_cached(&executable, platform.0, cache)?;
Ok(Self {
root: root.to_path_buf(),
interpreter_info,
})
}
/// Creating a new venv from a python interpreter changes this
pub fn new_prefix(venv: &Path, interpreter_info: &InterpreterInfo) -> Self {
Self {
root: venv.to_path_buf(),
interpreter_info: InterpreterInfo {
base_prefix: venv.to_path_buf(),
..interpreter_info.clone()
},
}
}
/// Returns the location of the python interpreter
pub fn python_executable(&self) -> PathBuf {
#[cfg(unix)]
{
self.root.join("bin").join("python")
}
#[cfg(windows)]
{
self.0
.join("Scripts")
.join("python.exe")
.into_std_path_buf()
}
#[cfg(not(any(unix, windows)))]
{
compile_error!("Only windows and unix (linux, mac os, etc.) are supported")
}
}
pub fn root(&self) -> &Path {
&self.root
}
pub fn interpreter_info(&self) -> &InterpreterInfo {
&self.interpreter_info
}
/// Returns the path to the `site-packages` directory inside a virtual environment.
pub fn site_packages(&self) -> PathBuf {
self.interpreter_info
.platform
.venv_site_packages(&self.root, self.interpreter_info().simple_version())
}
}
/// Locate the current virtual environment.
pub(crate) fn detect_virtual_env(target: &PythonPlatform) -> Result<PathBuf> {
match (env::var_os("VIRTUAL_ENV"), env::var_os("CONDA_PREFIX")) {

3
crates/puffin-resolver/Cargo.toml
View file

@ -10,7 +10,6 @@ authors = { workspace = true }
license = { workspace = true }
[dependencies]
gourgeist = { path = "../gourgeist" }
install-wheel-rs = { path = "../install-wheel-rs" }
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
@ -39,10 +38,10 @@ tokio-util = { workspace = true, features = ["compat"] }
tracing = { workspace = true }
url = { workspace = true }
waitmap = { workspace = true }
which = { workspace = true }
zip = { workspace = true }
[dev-dependencies]
gourgeist = { path = "../gourgeist" }
puffin-interpreter = { path = "../puffin-interpreter" }
once_cell = { version = "1.18.0" }

11
crates/puffin-resolver/tests/resolver.rs
View file

@ -9,14 +9,13 @@ use std::pin::Pin;
use std::str::FromStr;
use anyhow::Result;
use gourgeist::Venv;
use once_cell::sync::Lazy;
use pep508_rs::{MarkerEnvironment, Requirement, StringVersion};
use platform_host::{Arch, Os, Platform};
use platform_tags::Tags;
use puffin_client::RegistryClientBuilder;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_resolver::{ResolutionMode, Resolver};
use puffin_traits::BuildContext;
@ -27,7 +26,11 @@ impl BuildContext for DummyContext {
panic!("The test should not need to build source distributions")
}
fn python(&self) -> &PythonExecutable {
fn interpreter_info(&self) -> &InterpreterInfo {
panic!("The test should not need to build source distributions")
}
fn base_python(&self) -> &Path {
panic!("The test should not need to build source distributions")
}
@ -41,7 +44,7 @@ impl BuildContext for DummyContext {
fn install<'a>(
&'a self,
_requirements: &'a [Requirement],
_venv: &'a Venv,
_venv: &'a Virtualenv,
) -> Pin<Box<dyn Future<Output = Result<()>> + 'a>> {
panic!("The test should not need to build source distributions")
}

1
crates/puffin-traits/Cargo.toml
View file

@ -10,7 +10,6 @@ authors = { workspace = true }
license = { workspace = true }
[dependencies]
gourgeist = { path = "../gourgeist" }
pep508_rs = { path = "../pep508-rs" }
puffin-interpreter = { path = "../puffin-interpreter" }

9
crates/puffin-traits/src/lib.rs
View file

@ -4,9 +4,8 @@ use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use gourgeist::Venv;
use pep508_rs::Requirement;
use puffin_interpreter::PythonExecutable;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
/// Avoid cyclic crate dependencies between resolver, installer and builder.
///
@ -53,7 +52,9 @@ pub trait BuildContext {
/// All (potentially nested) source distribution builds use the same base python and can reuse
/// it's metadata (e.g. wheel compatibility tags).
fn python(&self) -> &PythonExecutable;
fn interpreter_info(&self) -> &InterpreterInfo;
/// The system (or conda) python interpreter to create venvs.
fn base_python(&self) -> &Path;
/// Resolve the given requirements into a ready-to-install set of package versions.
fn resolve<'a>(
@ -65,7 +66,7 @@ pub trait BuildContext {
fn install<'a>(
&'a self,
requirements: &'a [Requirement],
venv: &'a Venv,
venv: &'a Virtualenv,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + 'a>>;
/// Build a source distribution into a wheel from an archive.
///

4
ruff.toml Normal file
View file

@ -0,0 +1,4 @@
exclude = [
"crates/gourgeist/src/activator/activate_this.py",
"crates/gourgeist/src/_virtualenv.py"
]