diff --git a/Cargo.lock b/Cargo.lock index 77a8e23a0..a9ff0ae2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1003,9 +1003,10 @@ dependencies = [ "distribution-filename", "fs-err", "install-wheel-rs", + "platform-host", + "puffin-interpreter", "rayon", "reqwest", - "seahash", "serde", "serde_json", "tempfile", @@ -2021,6 +2022,7 @@ dependencies = [ "pep508_rs", "platform-host", "platform-tags", + "puffin-interpreter", "puffin-traits", "pyproject-toml", "serde", @@ -2180,7 +2182,9 @@ dependencies = [ "pep440_rs 0.3.12", "pep508_rs", "platform-host", + "serde", "serde_json", + "thiserror", "tokio", "tracing", ] @@ -2243,7 +2247,6 @@ dependencies = [ "tracing", "url", "waitmap", - "which", "zip", ] @@ -2252,7 +2255,6 @@ name = "puffin-traits" version = "0.1.0" dependencies = [ "anyhow", - "gourgeist", "pep508_rs", "puffin-interpreter", ] @@ -2708,12 +2710,6 @@ dependencies = [ "syn 2.0.38", ] -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "2.9.2" diff --git a/crates/gourgeist/Cargo.toml b/crates/gourgeist/Cargo.toml index 72a4c09b9..d2b5e1eb5 100644 --- a/crates/gourgeist/Cargo.toml +++ b/crates/gourgeist/Cargo.toml @@ -14,8 +14,10 @@ authors = { workspace = true } license = { workspace = true } [dependencies] -install-wheel-rs = { path = "../install-wheel-rs", optional = true } distribution-filename = { path = "../distribution-filename" } +install-wheel-rs = { path = "../install-wheel-rs", optional = true } +platform-host = { path = "../platform-host" } +puffin-interpreter = { path = "../puffin-interpreter" } camino = { workspace = true } clap = { workspace = true } @@ -24,7 +26,6 @@ dirs = { workspace = true } fs-err = { workspace = true } reqwest = { workspace = true, optional = true, features = ["blocking"] } rayon = { workspace = true, optional = true } -seahash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/gourgeist/src/bare.rs b/crates/gourgeist/src/bare.rs index fbcf204dc..679b6780f 100644 --- a/crates/gourgeist/src/bare.rs +++ b/crates/gourgeist/src/bare.rs @@ -8,10 +8,9 @@ use fs_err as fs; #[cfg(unix)] use fs_err::os::unix::fs::symlink; use fs_err::File; +use puffin_interpreter::InterpreterInfo; use tracing::info; -use crate::interpreter::InterpreterInfo; - /// The bash activate scripts with the venv dependent paths patches out const ACTIVATE_TEMPLATES: &[(&str, &str)] = &[ ("activate", include_str!("activator/activate")), @@ -111,10 +110,17 @@ pub(crate) fn create_bare_venv( #[cfg(unix)] { symlink(base_python, &venv_python)?; - symlink("python", bin_dir.join(format!("python{}", info.major)))?; symlink( "python", - bin_dir.join(format!("python{}.{}", info.major, info.minor)), + bin_dir.join(format!("python{}", info.simple_version().0)), + )?; + symlink( + "python", + bin_dir.join(format!( + "python{}.{}", + info.simple_version().0, + info.simple_version().1 + )), )?; } @@ -124,7 +130,11 @@ pub(crate) fn create_bare_venv( .replace("{{ VIRTUAL_ENV_DIR }}", location.as_str()) .replace( "{{ RELATIVE_SITE_PACKAGES }}", - &format!("../lib/python{}.{}/site-packages", info.major, info.minor), + &format!( + "../lib/python{}.{}/site-packages", + info.simple_version().0, + info.simple_version().1 + ), ); fs::write(bin_dir.join(name), activator)?; } @@ -142,12 +152,18 @@ pub(crate) fn create_bare_venv( let pyvenv_cfg_data = &[ ("home", python_home), ("implementation", "CPython".to_string()), - ("version_info", info.python_version.clone()), + ("version_info", info.markers().python_version.string.clone()), ("gourgeist", env!("CARGO_PKG_VERSION").to_string()), // I wouldn't allow this option anyway ("include-system-site-packages", "false".to_string()), - ("base-prefix", info.base_prefix.clone()), - ("base-exec-prefix", info.base_exec_prefix.clone()), + ( + "base-prefix", + info.base_prefix().to_string_lossy().to_string(), + ), + ( + "base-exec-prefix", + info.base_exec_prefix().to_string_lossy().to_string(), + ), ("base-executable", base_python.to_string()), ]; let mut pyvenv_cfg = BufWriter::new(File::create(location.join("pyvenv.cfg"))?); @@ -157,7 +173,11 @@ pub(crate) fn create_bare_venv( // TODO: This is different on windows let site_packages = location .join("lib") - .join(format!("python{}.{}", info.major, info.minor)) + .join(format!( + "python{}.{}", + info.simple_version().0, + info.simple_version().1 + )) .join("site-packages"); fs::create_dir_all(&site_packages)?; // Install _virtualenv.py patch. diff --git a/crates/gourgeist/src/interpreter.rs b/crates/gourgeist/src/interpreter.rs index bdddb2ac6..e7a271767 100644 --- a/crates/gourgeist/src/interpreter.rs +++ b/crates/gourgeist/src/interpreter.rs @@ -1,160 +1,5 @@ -use std::io; -use std::io::{BufReader, Write}; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::time::SystemTime; - -use camino::{Utf8Path, Utf8PathBuf}; -use fs_err as fs; -use fs_err::File; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error, warn}; - -use crate::{crate_cache_dir, Error}; - -const QUERY_PYTHON: &str = include_str!("query_python.py"); - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct InterpreterInfo { - pub base_exec_prefix: String, - pub base_prefix: String, - pub major: u8, - pub minor: u8, - pub python_version: String, -} - -/// Gets the interpreter.rs info, either cached or by running it. -pub fn get_interpreter_info(interpreter: impl AsRef) -> Result { - let interpreter = Utf8Path::from_path(interpreter.as_ref()) - .ok_or_else(|| Error::NonUTF8Path(interpreter.as_ref().to_path_buf()))?; - - let cache_dir = crate_cache_dir()?.join("interpreter_info"); - - let index = seahash::hash(interpreter.as_str().as_bytes()); - let cache_file = cache_dir.join(index.to_string()).with_extension("json"); - - let modified = fs::metadata(interpreter)? - .modified()? - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_millis(); - - if cache_file.exists() { - let cache_entry: Result = File::open(&cache_file) - .map_err(|err| err.to_string()) - .and_then(|cache_reader| { - serde_json::from_reader(BufReader::new(cache_reader)).map_err(|err| err.to_string()) - }); - match cache_entry { - Ok(cache_entry) => { - debug!("Using cache entry {cache_file}"); - if modified == cache_entry.modified && interpreter == cache_entry.interpreter { - return Ok(cache_entry.interpreter_info); - } - debug!( - "Removing mismatching cache entry {cache_file} ({} {} {} {})", - modified, cache_entry.modified, interpreter, cache_entry.interpreter - ); - if let Err(remove_err) = fs::remove_file(&cache_file) { - warn!("Failed to mismatching cache file at {cache_file}: {remove_err}"); - } - } - Err(cache_err) => { - debug!("Removing broken cache entry {cache_file} ({cache_err})"); - if let Err(remove_err) = fs::remove_file(&cache_file) { - warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})"); - } - } - } - } - - let interpreter_info = query_interpreter(interpreter)?; - fs::create_dir_all(&cache_dir)?; - let cache_entry = CacheEntry { - interpreter: interpreter.to_path_buf(), - modified, - interpreter_info: interpreter_info.clone(), - }; - let mut cache_writer = File::create(&cache_file)?; - serde_json::to_writer(&mut cache_writer, &cache_entry).map_err(io::Error::from)?; - - Ok(interpreter_info) -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct CacheEntry { - interpreter: Utf8PathBuf, - modified: u128, - interpreter_info: InterpreterInfo, -} - -/// Runs a python script that returns the relevant info about the interpreter.rs as json -fn query_interpreter(interpreter: &Utf8Path) -> Result { - let mut child = Command::new(interpreter) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - - if let Some(mut stdin) = child.stdin.take() { - stdin - .write_all(QUERY_PYTHON.as_bytes()) - .map_err(|err| Error::PythonSubcommand { - interpreter: interpreter.to_path_buf(), - err, - })?; - } - let output = child.wait_with_output()?; - let stdout = String::from_utf8(output.stdout).unwrap_or_else(|err| { - // At this point, there was most likely an error caused by a non-utf8 character, so we're in - // an ugly case but still very much want to give the user a chance - error!( - "The stdout of the failed call of the call to {} contains non-utf8 characters", - interpreter - ); - String::from_utf8_lossy(err.as_bytes()).to_string() - }); - let stderr = String::from_utf8(output.stderr).unwrap_or_else(|err| { - error!( - "The stderr of the failed call of the call to {} contains non-utf8 characters", - interpreter - ); - String::from_utf8_lossy(err.as_bytes()).to_string() - }); - // stderr isn't technically a criterion for success, but i don't know of any cases where there - // should be stderr output and if there is, we want to know - if !output.status.success() || !stderr.trim().is_empty() { - return Err(Error::PythonSubcommand { - interpreter: interpreter.to_path_buf(), - err: io::Error::new( - io::ErrorKind::Other, - format!( - "Querying python at {} failed with status {}:\n--- stdout:\n{}\n--- stderr:\n{}", - interpreter, - output.status, - stdout.trim(), - stderr.trim() - ), - ) - }); - } - let data = serde_json::from_str::(&stdout).map_err(|err| - Error::PythonSubcommand { - interpreter: interpreter.to_path_buf(), - err: io::Error::new( - io::ErrorKind::Other, - format!( - "Querying python at {} did not return the expected data ({}):\n--- stdout:\n{}\n--- stderr:\n{}", - interpreter, - err, - stdout.trim(), - stderr.trim() - ) - ) - } - )?; - Ok(data) -} +use camino::Utf8PathBuf; +use tracing::debug; /// Parse the value of the `-p`/`--python` option, which can be e.g. `3.11`, `python3.11`, /// `tools/bin/python3.11` or `/usr/bin/python3.11`. diff --git a/crates/gourgeist/src/lib.rs b/crates/gourgeist/src/lib.rs index 737cc6265..6f2905763 100644 --- a/crates/gourgeist/src/lib.rs +++ b/crates/gourgeist/src/lib.rs @@ -1,5 +1,4 @@ use std::io; -use std::ops::Deref; use std::path::{Path, PathBuf}; use camino::{Utf8Path, Utf8PathBuf}; @@ -7,7 +6,9 @@ use dirs::cache_dir; use tempfile::PersistError; use thiserror::Error; -pub use interpreter::{get_interpreter_info, parse_python_cli, InterpreterInfo}; +pub use interpreter::parse_python_cli; +use platform_host::PlatformError; +use puffin_interpreter::{InterpreterInfo, Virtualenv}; use crate::bare::create_bare_venv; @@ -53,44 +54,8 @@ pub enum Error { }, #[error("{0} is not a valid UTF-8 path")] NonUTF8Path(PathBuf), -} - -/// Provides the paths inside a venv -#[derive(Debug, Clone)] -pub struct Venv(Utf8PathBuf); - -impl Deref for Venv { - type Target = Utf8Path; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Venv { - pub fn new(location: impl Into) -> Result { - let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?; - Ok(Self(location)) - } - - /// Returns the location of the python interpreter - pub fn python_interpreter(&self) -> PathBuf { - #[cfg(unix)] - { - self.0.join("bin").join("python").into_std_path_buf() - } - #[cfg(windows)] - { - self.0 - .join("Scripts") - .join("python.exe") - .into_std_path_buf() - } - #[cfg(not(any(unix, windows)))] - { - compile_error!("Only windows and unix (linux, mac os, etc.) are supported") - } - } + #[error(transparent)] + Platform(#[from] PlatformError), } pub(crate) fn crate_cache_dir() -> io::Result { @@ -106,7 +71,7 @@ pub fn create_venv( base_python: impl AsRef, info: &InterpreterInfo, bare: bool, -) -> Result { +) -> Result { let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?; let base_python = Utf8Path::from_path(base_python.as_ref()) .ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?; @@ -128,5 +93,5 @@ pub fn create_venv( } } - Ok(Venv(location)) + Ok(Virtualenv::new_prefix(location.as_std_path(), info)) } diff --git a/crates/gourgeist/src/main.rs b/crates/gourgeist/src/main.rs index ef4c6cbc1..5c32e8a17 100644 --- a/crates/gourgeist/src/main.rs +++ b/crates/gourgeist/src/main.rs @@ -4,13 +4,14 @@ use std::time::Instant; use camino::Utf8PathBuf; use clap::Parser; +use gourgeist::{create_venv, parse_python_cli}; +use platform_host::Platform; +use puffin_interpreter::InterpreterInfo; use tracing::info; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; -use gourgeist::{create_venv, get_interpreter_info, parse_python_cli}; - #[derive(Parser, Debug)] struct Cli { path: Option, @@ -24,8 +25,9 @@ fn run() -> Result<(), gourgeist::Error> { let cli = Cli::parse(); let location = cli.path.unwrap_or(Utf8PathBuf::from(".venv")); let python = parse_python_cli(cli.python)?; - let data = get_interpreter_info(&python)?; - create_venv(location, &python, &data, cli.bare)?; + let platform = Platform::current()?; + let info = InterpreterInfo::query_cached(python.as_std_path(), platform, None).unwrap(); + create_venv(location, &python, &info, cli.bare)?; Ok(()) } diff --git a/crates/gourgeist/src/packages.rs b/crates/gourgeist/src/packages.rs index bfd6899e7..c656296b2 100644 --- a/crates/gourgeist/src/packages.rs +++ b/crates/gourgeist/src/packages.rs @@ -12,9 +12,9 @@ use tracing::debug; use distribution_filename::WheelFilename; use install_wheel_rs::{install_wheel, InstallLocation}; +use puffin_interpreter::InterpreterInfo; use crate::bare::VenvPaths; -use crate::interpreter::InterpreterInfo; use crate::{crate_cache_dir, Error}; pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result { @@ -51,7 +51,7 @@ pub(crate) fn install_base_packages( info: &InterpreterInfo, paths: &VenvPaths, ) -> Result<(), Error> { - let install_location = InstallLocation::new(location.canonicalize()?, (info.major, info.minor)); + let install_location = InstallLocation::new(location.canonicalize()?, info.simple_version()); let install_location = install_location.acquire_lock()?; // TODO: Use the json api instead diff --git a/crates/gourgeist/src/query_python.py b/crates/gourgeist/src/query_python.py deleted file mode 100644 index f71eab275..000000000 --- a/crates/gourgeist/src/query_python.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -import sys -from platform import python_version - - -def main(): - data = { - "base_exec_prefix": sys.base_exec_prefix, - "base_prefix": sys.base_prefix, - "major": sys.version_info.major, - "minor": sys.version_info.minor, - "python_version": python_version(), - } - print(json.dumps(data)) - - -if __name__ == "__main__": - main() diff --git a/crates/gourgeist/venv_checker.py b/crates/gourgeist/venv_checker.py index 839c48ba6..0f0b04a80 100644 --- a/crates/gourgeist/venv_checker.py +++ b/crates/gourgeist/venv_checker.py @@ -16,11 +16,7 @@ def main(): output = check_output(["bash"], input=command, text=True).strip() assert output.startswith("usage:"), output - output = ( - check_output([venv_python, "imasnake.py"], text=True) - .strip() - .splitlines() - ) + output = check_output([venv_python, "imasnake.py"], text=True).strip().splitlines() assert output[0] == str(project_root.joinpath(venv_python)), output assert not output[2].startswith(str(project_root)), output assert output[3] == str(project_root.joinpath(venv_name)), output diff --git a/crates/install-wheel-rs/src/install_location.rs b/crates/install-wheel-rs/src/install_location.rs index cd03c051d..e96448f39 100644 --- a/crates/install-wheel-rs/src/install_location.rs +++ b/crates/install-wheel-rs/src/install_location.rs @@ -75,14 +75,14 @@ impl AsRef for LockedDir { /// non-deterministically fail. pub struct InstallLocation { /// absolute path - venv_base: T, + venv_root: T, python_version: (u8, u8), } impl> InstallLocation { pub fn new(venv_base: T, python_version: (u8, u8)) -> Self { Self { - venv_base, + venv_root: venv_base, python_version, } } @@ -90,10 +90,10 @@ impl> InstallLocation { /// Returns the location of the `python` interpreter. pub fn python(&self) -> PathBuf { if cfg!(windows) { - self.venv_base.as_ref().join("Scripts").join("python.exe") + self.venv_root.as_ref().join("Scripts").join("python.exe") } else { // canonicalize on python would resolve the symlink - self.venv_base.as_ref().join("bin").join("python") + self.venv_root.as_ref().join("bin").join("python") } } @@ -101,25 +101,25 @@ impl> InstallLocation { self.python_version } - pub fn venv_base(&self) -> &T { - &self.venv_base + pub fn venv_root(&self) -> &T { + &self.venv_root } } impl InstallLocation { pub fn acquire_lock(&self) -> io::Result> { - let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_base)? { + let locked_dir = if let Some(locked_dir) = LockedDir::try_acquire(&self.venv_root)? { locked_dir } else { warn!( "Could not acquire exclusive lock for installing, is another installation process \ running? Sleeping until lock becomes free" ); - LockedDir::acquire(&self.venv_base)? + LockedDir::acquire(&self.venv_root)? }; Ok(InstallLocation { - venv_base: locked_dir, + venv_root: locked_dir, python_version: self.python_version, }) } diff --git a/crates/install-wheel-rs/src/linker.rs b/crates/install-wheel-rs/src/linker.rs index bd26277f7..fd73e183d 100644 --- a/crates/install-wheel-rs/src/linker.rs +++ b/crates/install-wheel-rs/src/linker.rs @@ -29,7 +29,7 @@ pub fn install_wheel( wheel: impl AsRef, link_mode: LinkMode, ) -> Result<(), Error> { - let base_location = location.venv_base(); + let root = location.venv_root(); // TODO(charlie): Pass this in. let site_packages_python = format!( @@ -38,10 +38,9 @@ pub fn install_wheel( location.python_version().1 ); let site_packages = if cfg!(target_os = "windows") { - base_location.as_ref().join("Lib").join("site-packages") + root.as_ref().join("Lib").join("site-packages") } else { - base_location - .as_ref() + root.as_ref() .join("lib") .join(site_packages_python) .join("site-packages") @@ -88,7 +87,7 @@ pub fn install_wheel( if data_dir.is_dir() { debug!(name, "Installing data"); install_data( - base_location.as_ref(), + root.as_ref(), &site_packages, &data_dir, &name, diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index 3f41ed5cb..3345a3789 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -722,7 +722,7 @@ fn install_script( /// Move the files from the .data directory to the right location in the venv #[allow(clippy::too_many_arguments)] pub(crate) fn install_data( - venv_base: &Path, + venv_root: &Path, site_packages: &Path, data_dir: &Path, dist_name: &str, @@ -736,7 +736,7 @@ pub(crate) fn install_data( match data_entry.file_name().as_os_str().to_str() { Some("data") => { // Move the content of the folder to the root of the venv - move_folder_recorded(&data_entry.path(), venv_base, site_packages, record)?; + move_folder_recorded(&data_entry.path(), venv_root, site_packages, record)?; } Some("scripts") => { for file in fs::read_dir(data_entry.path())? { @@ -762,7 +762,7 @@ pub(crate) fn install_data( } } Some("headers") => { - let target_path = venv_base + let target_path = venv_root .join("include") .join("site") .join(format!( @@ -901,7 +901,7 @@ pub fn install_wheel( let name = &filename.distribution; let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str()); - let base_location = location.venv_base(); + let base_location = location.venv_root(); let site_packages_python = format!( "python{}.{}", diff --git a/crates/pep440-rs/python/pep440_rs/__init__.pyi b/crates/pep440-rs/python/pep440_rs/__init__.pyi index c4796455c..095c65bd1 100644 --- a/crates/pep440-rs/python/pep440_rs/__init__.pyi +++ b/crates/pep440-rs/python/pep440_rs/__init__.pyi @@ -1,7 +1,6 @@ # Generated by `stubgen -p pep440_rs` from typing import Any, ClassVar - class Version: dev: Any epoch: Any @@ -24,7 +23,6 @@ class Version: def __lt__(self, other) -> Any: ... def __ne__(self, other) -> Any: ... - class VersionSpecifier: __hash__: ClassVar[None] = ... diff --git a/crates/puffin-build-cli/src/main.rs b/crates/puffin-build-cli/src/main.rs index 53208a233..d89d736ce 100644 --- a/crates/puffin-build-cli/src/main.rs +++ b/crates/puffin-build-cli/src/main.rs @@ -20,7 +20,7 @@ use platform_host::Platform; use puffin_build::SourceDistributionBuilder; use puffin_client::RegistryClientBuilder; use puffin_dispatch::BuildDispatch; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; #[derive(Parser)] struct Args { @@ -44,17 +44,22 @@ async fn run() -> Result<()> { }; let dirs = ProjectDirs::from("", "", "puffin"); - let cache = dirs.as_ref().map(ProjectDirs::cache_dir); + let cache = dirs + .as_ref() + .map(|dir| ProjectDirs::cache_dir(dir).to_path_buf()); let platform = Platform::current()?; - let python = PythonExecutable::from_env(platform, cache)?; + let venv = Virtualenv::from_env(platform, cache.as_deref())?; - let interpreter_info = gourgeist::get_interpreter_info(python.executable())?; - - let build_dispatch = - BuildDispatch::new(RegistryClientBuilder::default().build(), python, cache); + let build_dispatch = BuildDispatch::new( + RegistryClientBuilder::default().build(), + cache, + venv.interpreter_info().clone(), + fs::canonicalize(venv.python_executable())?, + ); let builder = - SourceDistributionBuilder::setup(&args.sdist, &interpreter_info, &build_dispatch).await?; + SourceDistributionBuilder::setup(&args.sdist, venv.interpreter_info(), &build_dispatch) + .await?; let wheel = builder.build(&wheel_dir)?; println!("Wheel built to {}", wheel_dir.join(wheel).display()); Ok(()) diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index f8ebae19d..509f8310d 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -15,6 +15,7 @@ gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } +puffin-interpreter = { path = "../puffin-interpreter" } puffin-traits = { path = "../puffin-traits" } anyhow = { workspace = true } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index b21ccf033..d39108d2f 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -19,8 +19,8 @@ use thiserror::Error; use tracing::{debug, instrument}; use zip::ZipArchive; -use gourgeist::{InterpreterInfo, Venv}; use pep508_rs::Requirement; +use puffin_interpreter::{InterpreterInfo, Virtualenv}; use puffin_traits::BuildContext; #[derive(Error, Debug)] @@ -88,7 +88,7 @@ pub struct SourceDistributionBuilder { source_tree: PathBuf, /// `Some` if this is a PEP 517 build pep517_backend: Option, - venv: Venv, + venv: Virtualenv, /// Populated if `prepare_metadata_for_build_wheel` was called. /// /// > If the build frontend has previously called prepare_metadata_for_build_wheel and depends @@ -154,7 +154,7 @@ impl SourceDistributionBuilder { } let venv = gourgeist::create_venv( temp_dir.path().join("venv"), - build_context.python().executable(), + build_context.base_python(), interpreter_info, true, )?; @@ -211,8 +211,7 @@ impl SourceDistributionBuilder { print() "#, pep517_backend.backend_import(), escape_path_for_python(&metadata_directory) }; - let output = - run_python_script(&self.venv.python_interpreter(), &script, &self.source_tree)?; + let output = run_python_script(&self.venv.python_executable(), &script, &self.source_tree)?; if !output.status.success() { return Err(Error::from_command_output( "Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(), @@ -259,7 +258,7 @@ impl SourceDistributionBuilder { self.pep517_build_wheel(&wheel_dir, pep517_backend) } else { // We checked earlier that setup.py exists - let python_interpreter = self.venv.python_interpreter(); + let python_interpreter = self.venv.python_executable(); let output = Command::new(&python_interpreter) .args(["setup.py", "bdist_wheel"]) .current_dir(&self.source_tree) @@ -310,8 +309,7 @@ impl SourceDistributionBuilder { print(backend.build_wheel("{}", metadata_directory={})) "#, pep517_backend.backend_import(), escaped_wheel_dir, metadata_directory }; - let output = - run_python_script(&self.venv.python_interpreter(), &script, &self.source_tree)?; + let output = run_python_script(&self.venv.python_executable(), &script, &self.source_tree)?; if !output.status.success() { return Err(Error::from_command_output( "Build backend failed to build wheel through `build_wheel()` ".to_string(), @@ -346,13 +344,8 @@ async fn create_pep517_build_environment( data: &InterpreterInfo, pep517_backend: &Pep517Backend, build_context: &impl BuildContext, -) -> Result { - let venv = gourgeist::create_venv( - root.join(".venv"), - build_context.python().executable(), - data, - true, - )?; +) -> Result { + let venv = gourgeist::create_venv(root.join(".venv"), build_context.base_python(), data, true)?; let resolved_requirements = build_context .resolve(&pep517_backend.requirements) .await @@ -377,7 +370,7 @@ async fn create_pep517_build_environment( print(json.dumps(requires)) "#, pep517_backend.backend_import() }; - let output = run_python_script(&venv.python_interpreter(), &script, source_tree)?; + let output = run_python_script(&venv.python_executable(), &script, source_tree)?; if !output.status.success() { return Err(Error::from_command_output( "Build backend failed to determine extras requires with `get_requires_for_build_wheel`" diff --git a/crates/puffin-cli/src/commands/freeze.rs b/crates/puffin-cli/src/commands/freeze.rs index 4e92148c0..ef95b3d18 100644 --- a/crates/puffin-cli/src/commands/freeze.rs +++ b/crates/puffin-cli/src/commands/freeze.rs @@ -5,7 +5,7 @@ use tracing::debug; use platform_host::Platform; use puffin_installer::SitePackages; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use crate::commands::ExitStatus; use crate::printer::Printer; @@ -14,10 +14,10 @@ use crate::printer::Printer; pub(crate) fn freeze(cache: Option<&Path>, _printer: Printer) -> Result { // Detect the current Python interpreter. let platform = Platform::current()?; - let python = PythonExecutable::from_env(platform, cache)?; + let python = Virtualenv::from_env(platform, cache)?; debug!( "Using Python interpreter: {}", - python.executable().display() + python.python_executable().display() ); // Build the installed index. diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 44b74966f..80c7935b4 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -1,7 +1,7 @@ -use std::env; use std::fmt::Write; use std::io::{stdout, BufWriter}; use std::path::Path; +use std::{env, fs}; use anyhow::Result; use colored::Colorize; @@ -13,7 +13,7 @@ use platform_tags::Tags; use pubgrub::report::Reporter; use puffin_client::RegistryClientBuilder; use puffin_dispatch::BuildDispatch; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use puffin_resolver::ResolutionMode; use tracing::debug; @@ -50,16 +50,19 @@ pub(crate) async fn pip_compile( // Detect the current Python interpreter. let platform = Platform::current()?; - let python = PythonExecutable::from_env(platform, cache)?; + let venv = Virtualenv::from_env(platform, cache)?; debug!( "Using Python {} at {}", - python.markers().python_version, - python.executable().display() + venv.interpreter_info().markers().python_version, + venv.python_executable().display() ); // Determine the compatible platform tags. - let tags = Tags::from_env(python.platform(), python.simple_version())?; + let tags = Tags::from_env( + venv.interpreter_info().platform(), + venv.interpreter_info().simple_version(), + )?; // Instantiate a client. let client = { @@ -78,8 +81,9 @@ pub(crate) async fn pip_compile( let build_dispatch = BuildDispatch::new( RegistryClientBuilder::default().build(), - python.clone(), - cache, + cache.map(Path::to_path_buf), + venv.interpreter_info().clone(), + fs::canonicalize(venv.python_executable())?, ); // Resolve the dependencies. @@ -87,7 +91,7 @@ pub(crate) async fn pip_compile( requirements, constraints, mode, - python.markers(), + venv.interpreter_info().markers(), &tags, &client, &build_dispatch, diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index b04cb82bf..88a808e3a 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -12,7 +12,7 @@ use platform_host::Platform; use platform_tags::Tags; use puffin_client::RegistryClientBuilder; use puffin_installer::{Distribution, PartitionedRequirements, RemoteDistribution}; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use crate::commands::reporters::{ DownloadReporter, InstallReporter, UnzipReporter, WheelFinderReporter, @@ -58,10 +58,10 @@ pub(crate) async fn sync_requirements( // Detect the current Python interpreter. let platform = Platform::current()?; - let python = PythonExecutable::from_env(platform, cache)?; + let venv = Virtualenv::from_env(platform, cache)?; debug!( "Using Python interpreter: {}", - python.executable().display() + venv.python_executable().display() ); // Partition into those that should be linked from the cache (`local`), those that need to be @@ -70,7 +70,7 @@ pub(crate) async fn sync_requirements( local, remote, extraneous, - } = PartitionedRequirements::try_from_requirements(requirements, cache, &python)?; + } = PartitionedRequirements::try_from_requirements(requirements, cache, &venv)?; // Nothing to do. if remote.is_empty() && local.is_empty() && extraneous.is_empty() { @@ -90,7 +90,10 @@ pub(crate) async fn sync_requirements( } // Determine the current environment markers. - let tags = Tags::from_env(python.platform(), python.simple_version())?; + let tags = Tags::from_env( + venv.interpreter_info().platform(), + venv.interpreter_info().simple_version(), + )?; // Instantiate a client. let client = { @@ -226,7 +229,7 @@ pub(crate) async fn sync_requirements( let wheels = unzips.into_iter().chain(local).collect::>(); if !wheels.is_empty() { let start = std::time::Instant::now(); - puffin_installer::Installer::new(&python) + puffin_installer::Installer::new(&venv) .with_link_mode(link_mode) .with_reporter(InstallReporter::from(printer).with_length(wheels.len() as u64)) .install(&wheels)?; diff --git a/crates/puffin-cli/src/commands/pip_uninstall.rs b/crates/puffin-cli/src/commands/pip_uninstall.rs index 9c3f4dccb..677c01fb6 100644 --- a/crates/puffin-cli/src/commands/pip_uninstall.rs +++ b/crates/puffin-cli/src/commands/pip_uninstall.rs @@ -8,7 +8,7 @@ use tracing::debug; use pep508_rs::Requirement; use platform_host::Platform; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use puffin_package::package_name::PackageName; use crate::commands::{elapsed, ExitStatus}; @@ -32,14 +32,14 @@ pub(crate) async fn pip_uninstall( // Detect the current Python interpreter. let platform = Platform::current()?; - let python = PythonExecutable::from_env(platform, cache)?; + let venv = Virtualenv::from_env(platform, cache)?; debug!( "Using Python interpreter: {}", - python.executable().display() + venv.python_executable().display() ); // Index the current `site-packages` directory. - let site_packages = puffin_installer::SitePackages::try_from_executable(&python)?; + let site_packages = puffin_installer::SitePackages::try_from_executable(&venv)?; // Sort and deduplicate the requirements. let packages = { diff --git a/crates/puffin-cli/src/commands/venv.rs b/crates/puffin-cli/src/commands/venv.rs index bc855e0d4..a523cfc1d 100644 --- a/crates/puffin-cli/src/commands/venv.rs +++ b/crates/puffin-cli/src/commands/venv.rs @@ -5,6 +5,8 @@ use anyhow::Result; use colored::Colorize; use fs_err as fs; use miette::{Diagnostic, IntoDiagnostic}; +use platform_host::Platform; +use puffin_interpreter::InterpreterInfo; use thiserror::Error; use crate::commands::ExitStatus; @@ -37,7 +39,7 @@ enum VenvError { #[error("Failed to extract Python interpreter info")] #[diagnostic(code(puffin::venv::interpreter))] - InterpreterError(#[source] gourgeist::Error), + InterpreterError(#[source] anyhow::Error), #[error("Failed to create virtual environment")] #[diagnostic(code(puffin::venv::creation))] @@ -59,8 +61,10 @@ fn venv_impl( .or_else(|_| which::which("python")) .map_err(|_| VenvError::PythonNotFound)? }; - let interpreter_info = - gourgeist::get_interpreter_info(&base_python).map_err(VenvError::InterpreterError)?; + let platform = Platform::current().into_diagnostic()?; + // TODO(konstin): Add caching + let interpreter_info = InterpreterInfo::query_cached(&base_python, platform, None) + .map_err(VenvError::InterpreterError)?; writeln!( printer, diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index 0a4400bf8..d1de5ece3 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -10,7 +10,6 @@ use anyhow::Context; use itertools::Itertools; use tempfile::tempdir; -use gourgeist::Venv; use pep508_rs::Requirement; use platform_tags::Tags; use puffin_build::SourceDistributionBuilder; @@ -18,7 +17,7 @@ use puffin_client::RegistryClient; use puffin_installer::{ uninstall, Downloader, Installer, PartitionedRequirements, RemoteDistribution, Unzipper, }; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::{InterpreterInfo, Virtualenv}; use puffin_resolver::{ResolutionMode, Resolver, WheelFinder}; use puffin_traits::BuildContext; use tracing::debug; @@ -27,19 +26,23 @@ use tracing::debug; /// documentation. pub struct BuildDispatch { client: RegistryClient, - python: PythonExecutable, cache: Option, + interpreter_info: InterpreterInfo, + base_python: PathBuf, } impl BuildDispatch { - pub fn new(client: RegistryClient, python: PythonExecutable, cache: Option) -> Self - where - T: Into, - { + pub fn new( + client: RegistryClient, + cache: Option, + interpreter_info: InterpreterInfo, + base_python: PathBuf, + ) -> Self { Self { client, - python, - cache: cache.map(Into::into), + cache, + interpreter_info, + base_python, } } } @@ -49,8 +52,12 @@ impl BuildContext for BuildDispatch { self.cache.as_deref() } - fn python(&self) -> &PythonExecutable { - &self.python + fn interpreter_info(&self) -> &InterpreterInfo { + &self.interpreter_info + } + + fn base_python(&self) -> &Path { + &self.base_python } fn resolve<'a>( @@ -58,12 +65,15 @@ impl BuildContext for BuildDispatch { requirements: &'a [Requirement], ) -> Pin>> + 'a>> { Box::pin(async { - let tags = Tags::from_env(self.python.platform(), self.python.simple_version())?; + let tags = Tags::from_env( + self.interpreter_info.platform(), + self.interpreter_info.simple_version(), + )?; let resolver = Resolver::new( requirements.to_vec(), Vec::default(), ResolutionMode::Highest, - self.python.markers(), + self.interpreter_info.markers(), &tags, &self.client, self, @@ -78,25 +88,20 @@ impl BuildContext for BuildDispatch { fn install<'a>( &'a self, requirements: &'a [Requirement], - venv: &'a Venv, + venv: &'a Virtualenv, ) -> Pin> + 'a>> { Box::pin(async move { debug!( "Install in {} requirements {}", - venv.as_str(), + venv.root().display(), requirements.iter().map(ToString::to_string).join(", ") ); - let python = self.python().with_venv(venv.as_std_path()); let PartitionedRequirements { local, remote, extraneous, - } = PartitionedRequirements::try_from_requirements( - requirements, - self.cache(), - &python, - )?; + } = PartitionedRequirements::try_from_requirements(requirements, self.cache(), venv)?; if !extraneous.is_empty() { debug!( @@ -117,7 +122,10 @@ impl BuildContext for BuildDispatch { remote.iter().map(ToString::to_string).join(", ") ); - let tags = Tags::from_env(python.platform(), python.simple_version())?; + let tags = Tags::from_env( + self.interpreter_info.platform(), + self.interpreter_info.simple_version(), + )?; let resolution = WheelFinder::new(&tags, &self.client) .resolve(&remote) .await?; @@ -144,7 +152,7 @@ impl BuildContext for BuildDispatch { .join(", ") ); let wheels = unzips.into_iter().chain(local).collect::>(); - Installer::new(&python).install(&wheels)?; + Installer::new(venv).install(&wheels)?; Ok(()) }) } @@ -155,8 +163,8 @@ impl BuildContext for BuildDispatch { wheel_dir: &'a Path, ) -> Pin> + 'a>> { Box::pin(async move { - let interpreter_info = gourgeist::get_interpreter_info(self.python.executable())?; - let builder = SourceDistributionBuilder::setup(sdist, &interpreter_info, self).await?; + let builder = + SourceDistributionBuilder::setup(sdist, &self.interpreter_info, self).await?; Ok(builder.build(wheel_dir)?) }) } diff --git a/crates/puffin-installer/src/installer.rs b/crates/puffin-installer/src/installer.rs index 84f327734..f8d7df607 100644 --- a/crates/puffin-installer/src/installer.rs +++ b/crates/puffin-installer/src/installer.rs @@ -2,22 +2,22 @@ use anyhow::{Error, Result}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use pep440_rs::Version; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use puffin_package::package_name::PackageName; use crate::CachedDistribution; pub struct Installer<'a> { - python: &'a PythonExecutable, + venv: &'a Virtualenv, link_mode: install_wheel_rs::linker::LinkMode, reporter: Option>, } impl<'a> Installer<'a> { /// Initialize a new installer. - pub fn new(python: &'a PythonExecutable) -> Self { + pub fn new(venv: &'a Virtualenv) -> Self { Self { - python, + venv, link_mode: install_wheel_rs::linker::LinkMode::default(), reporter: None, } @@ -43,8 +43,8 @@ impl<'a> Installer<'a> { tokio::task::block_in_place(|| { wheels.par_iter().try_for_each(|wheel| { let location = install_wheel_rs::InstallLocation::new( - self.python.venv().to_path_buf(), - self.python.simple_version(), + self.venv.root(), + self.venv.interpreter_info().simple_version(), ); install_wheel_rs::linker::install_wheel(&location, wheel.path(), self.link_mode)?; diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs index 2a022b94d..765de975d 100644 --- a/crates/puffin-installer/src/plan.rs +++ b/crates/puffin-installer/src/plan.rs @@ -4,7 +4,7 @@ use anyhow::Result; use tracing::debug; use pep508_rs::Requirement; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::Virtualenv; use puffin_package::package_name::PackageName; use crate::{CachedDistribution, InstalledDistribution, LocalIndex, SitePackages}; @@ -30,10 +30,10 @@ impl PartitionedRequirements { pub fn try_from_requirements( requirements: &[Requirement], cache: Option<&Path>, - python: &PythonExecutable, + venv: &Virtualenv, ) -> Result { // Index all the already-installed packages in site-packages. - let mut site_packages = SitePackages::try_from_executable(python)?; + let mut site_packages = SitePackages::try_from_executable(venv)?; // Index all the already-downloaded wheels in the cache. let local_index = if let Some(cache) = cache { diff --git a/crates/puffin-installer/src/site_packages.rs b/crates/puffin-installer/src/site_packages.rs index 50bda7e0f..aba9ec653 100644 --- a/crates/puffin-installer/src/site_packages.rs +++ b/crates/puffin-installer/src/site_packages.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use anyhow::Result; use fs_err as fs; +use puffin_interpreter::Virtualenv; -use puffin_interpreter::PythonExecutable; use puffin_package::package_name::PackageName; use crate::InstalledDistribution; @@ -13,10 +13,10 @@ pub struct SitePackages(BTreeMap); impl SitePackages { /// Build an index of installed packages from the given Python executable. - pub fn try_from_executable(python: &PythonExecutable) -> Result { + pub fn try_from_executable(venv: &Virtualenv) -> Result { let mut index = BTreeMap::new(); - for entry in fs::read_dir(python.site_packages())? { + for entry in fs::read_dir(venv.site_packages())? { let entry = entry?; if entry.file_type()?.is_dir() { if let Some(dist_info) = InstalledDistribution::try_from_path(&entry.path())? { diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index c9708a971..049052f89 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -11,12 +11,14 @@ license = { workspace = true } [dependencies] pep440_rs = { path = "../pep440-rs" } -pep508_rs = { path = "../pep508-rs" } +pep508_rs = { path = "../pep508-rs", features = ["serde"] } platform-host = { path = "../platform-host" } anyhow = { workspace = true } cacache = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +serde = { workspace = true, features = ["derive"] } diff --git a/crates/puffin-interpreter/src/get_interpreter_info.py b/crates/puffin-interpreter/src/get_interpreter_info.py new file mode 100644 index 000000000..26419ff9b --- /dev/null +++ b/crates/puffin-interpreter/src/get_interpreter_info.py @@ -0,0 +1,39 @@ +import json +import os +import platform +import sys + + +def format_full_version(info): + version = "{0.major}.{0.minor}.{0.micro}".format(info) + kind = info.releaselevel + if kind != "final": + version += kind[0] + str(info.serial) + return version + + +if hasattr(sys, "implementation"): + implementation_version = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name +else: + implementation_version = "0" + implementation_name = "" +markers = { + "implementation_name": implementation_name, + "implementation_version": implementation_version, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_python_implementation": platform.python_implementation(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "python_version": ".".join(platform.python_version_tuple()[:2]), + "sys_platform": sys.platform, +} +interpreter_info = { + "markers": markers, + "base_prefix": sys.base_prefix, + "base_exec_prefix": sys.base_exec_prefix, +} +print(json.dumps(interpreter_info)) diff --git a/crates/puffin-interpreter/src/interpreter_info.rs b/crates/puffin-interpreter/src/interpreter_info.rs new file mode 100644 index 000000000..0c2624726 --- /dev/null +++ b/crates/puffin-interpreter/src/interpreter_info.rs @@ -0,0 +1,189 @@ +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use pep440_rs::Version; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tracing::debug; + +use crate::python_platform::PythonPlatform; +use pep508_rs::MarkerEnvironment; +use platform_host::Platform; + +/// A Python executable and its associated platform markers. +#[derive(Debug, Clone)] +pub struct InterpreterInfo { + pub(crate) platform: PythonPlatform, + pub(crate) markers: MarkerEnvironment, + pub(crate) base_exec_prefix: PathBuf, + pub(crate) base_prefix: PathBuf, +} + +impl InterpreterInfo { + pub fn query_cached( + executable: &Path, + platform: Platform, + cache: Option<&Path>, + ) -> anyhow::Result { + let info = InterpreterQueryResult::query_cached(executable, cache)?; + debug_assert!( + info.base_prefix == info.base_exec_prefix, + "Not a venv python: {}, prefix: {}", + executable.display(), + info.base_prefix.display() + ); + + Ok(Self { + platform: PythonPlatform(platform), + markers: info.markers, + base_exec_prefix: info.base_exec_prefix, + base_prefix: info.base_prefix, + }) + } +} + +impl InterpreterInfo { + /// Returns the path to the Python virtual environment. + pub fn platform(&self) -> &Platform { + &self.platform + } + + /// Returns the [`MarkerEnvironment`] for this Python executable. + pub fn markers(&self) -> &MarkerEnvironment { + &self.markers + } + + /// Returns the Python version. + pub fn version(&self) -> &Version { + &self.markers.python_version.version + } + + /// Returns the Python version as a simple tuple. + pub fn simple_version(&self) -> (u8, u8) { + ( + u8::try_from(self.version().release[0]).expect("invalid major version"), + u8::try_from(self.version().release[1]).expect("invalid minor version"), + ) + } + + pub fn base_exec_prefix(&self) -> &Path { + &self.base_exec_prefix + } + pub fn base_prefix(&self) -> &Path { + &self.base_prefix + } +} + +#[derive(Debug, Error)] +pub(crate) enum InterpreterQueryError { + #[error(transparent)] + IO(#[from] io::Error), + #[error("Failed to query python interpreter at {interpreter}")] + PythonSubcommand { + interpreter: PathBuf, + #[source] + err: io::Error, + }, +} + +#[derive(Deserialize, Serialize)] +pub(crate) struct InterpreterQueryResult { + pub(crate) markers: MarkerEnvironment, + pub(crate) base_exec_prefix: PathBuf, + pub(crate) base_prefix: PathBuf, +} + +impl InterpreterQueryResult { + /// Return the resolved [`InterpreterQueryResult`] for the given Python executable. + pub(crate) fn query(interpreter: &Path) -> Result { + let output = Command::new(interpreter) + .args(["-c", include_str!("get_interpreter_info.py")]) + .output() + .map_err(|err| InterpreterQueryError::PythonSubcommand { + interpreter: interpreter.to_path_buf(), + err, + })?; + + // stderr isn't technically a criterion for success, but i don't know of any cases where there + // should be stderr output and if there is, we want to know + if !output.status.success() || !output.stderr.is_empty() { + return Err(InterpreterQueryError::PythonSubcommand { + interpreter: interpreter.to_path_buf(), + err: io::Error::new( + io::ErrorKind::Other, + format!( + "Querying python at {} failed with status {}:\n--- stdout:\n{}\n--- stderr:\n{}", + interpreter.display(), + output.status, + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ), + ) + }); + } + let data = serde_json::from_slice::(&output.stdout).map_err(|err| + InterpreterQueryError::PythonSubcommand { + interpreter: interpreter.to_path_buf(), + err: io::Error::new( + io::ErrorKind::Other, + format!( + "Querying python at {} did not return the expected data ({}):\n--- stdout:\n{}\n--- stderr:\n{}", + interpreter.display(), + err, + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ) + ) + } + )?; + + Ok(data) + } + + /// A wrapper around [`markers::query_interpreter_info`] to cache the computed markers. + /// + /// Running a Python script is (relatively) expensive, and the markers won't change + /// unless the Python executable changes, so we use the executable's last modified + /// time as a cache key. + pub(crate) fn query_cached(executable: &Path, cache: Option<&Path>) -> anyhow::Result { + // Read from the cache. + let key = if let Some(cache) = cache { + if let Ok(key) = cache_key(executable) { + if let Ok(data) = cacache::read_sync(cache, &key) { + debug!("Using cached markers for {}", executable.display()); + return Ok(serde_json::from_slice::(&data)?); + } + Some(key) + } else { + None + } + } else { + None + }; + + // Otherwise, run the Python script. + debug!("Detecting markers for {}", executable.display()); + let info = Self::query(executable)?; + + // Write to the cache. + if let Some(cache) = cache { + if let Some(key) = key { + cacache::write_sync(cache, key, serde_json::to_vec(&info)?)?; + } + } + + Ok(info) + } +} + +/// Create a cache key for the Python executable, consisting of the executable's +/// last modified time and the executable's path. +fn cache_key(executable: &Path) -> anyhow::Result { + let modified = executable + .metadata()? + .modified()? + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(); + Ok(format!("puffin:v0:{}:{}", executable.display(), modified)) +} diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index e26b23062..056fc9c34 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -1,103 +1,6 @@ -use std::path::{Path, PathBuf}; +pub use crate::interpreter_info::InterpreterInfo; +pub use crate::virtual_env::Virtualenv; -use anyhow::Result; - -use pep440_rs::Version; -use pep508_rs::MarkerEnvironment; -use platform_host::Platform; - -use crate::python_platform::PythonPlatform; - -mod markers; +mod interpreter_info; mod python_platform; mod virtual_env; - -/// A Python executable and its associated platform markers. -#[derive(Debug, Clone)] -pub struct PythonExecutable { - platform: PythonPlatform, - venv: PathBuf, - executable: PathBuf, - markers: MarkerEnvironment, -} - -impl PythonExecutable { - /// Detect the current Python executable from the host environment. - pub fn from_env(platform: Platform, cache: Option<&Path>) -> Result { - let platform = PythonPlatform::from(platform); - let venv = virtual_env::detect_virtual_env(&platform)?; - let executable = platform.venv_python(&venv); - let markers = markers::detect_cached_markers(&executable, cache)?; - - Ok(Self { - platform, - venv, - executable, - markers, - }) - } - - pub fn from_venv(platform: Platform, venv: &Path, cache: Option<&Path>) -> Result { - let platform = PythonPlatform::from(platform); - let executable = platform.venv_python(venv); - let markers = markers::detect_cached_markers(&executable, cache)?; - - Ok(Self { - platform, - venv: venv.to_path_buf(), - executable, - markers, - }) - } - - /// Create a [`PythonExecutable`] for a venv with a known base [`PythonExecutable`]. - #[must_use] - pub fn with_venv(&self, venv: &Path) -> Self { - let executable = self.platform.venv_python(venv); - - Self { - venv: venv.to_path_buf(), - executable, - ..self.clone() - } - } - - /// Returns the path to the Python virtual environment. - pub fn platform(&self) -> &Platform { - &self.platform - } - - /// Returns the path to the `site-packages` directory inside a virtual environment. - pub fn site_packages(&self) -> PathBuf { - self.platform - .venv_site_packages(self.venv(), self.simple_version()) - } - - /// Returns the path to the Python virtual environment. - pub fn venv(&self) -> &Path { - self.venv.as_path() - } - - /// Returns the path to the Python executable. - pub fn executable(&self) -> &Path { - self.executable.as_path() - } - - /// Returns the [`MarkerEnvironment`] for this Python executable. - pub fn markers(&self) -> &MarkerEnvironment { - &self.markers - } - - /// Returns the Python version. - pub fn version(&self) -> &Version { - &self.markers.python_version.version - } - - /// Returns the Python version as a simple tuple. - pub fn simple_version(&self) -> (u8, u8) { - ( - u8::try_from(self.version().release[0]).expect("invalid major version"), - u8::try_from(self.version().release[1]).expect("invalid minor version"), - ) - } -} diff --git a/crates/puffin-interpreter/src/markers.rs b/crates/puffin-interpreter/src/markers.rs deleted file mode 100644 index ac36e1a9a..000000000 --- a/crates/puffin-interpreter/src/markers.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::ffi::OsStr; -use std::path::Path; -use std::process::{Command, Output}; - -use anyhow::{Context, Result}; -use tracing::debug; - -use pep508_rs::MarkerEnvironment; - -/// Return the resolved [`MarkerEnvironment`] for the given Python executable. -pub(crate) fn detect_markers(python: impl AsRef) -> Result { - let output = call_python(python.as_ref(), ["-c", CAPTURE_MARKERS_SCRIPT])?; - Ok(serde_json::from_slice::(&output.stdout)?) -} - -/// A wrapper around [`markers::detect_markers`] to cache the computed markers. -/// -/// Running a Python script is (relatively) expensive, and the markers won't change -/// unless the Python executable changes, so we use the executable's last modified -/// time as a cache key. -pub(crate) fn detect_cached_markers( - executable: &Path, - cache: Option<&Path>, -) -> Result { - // Read from the cache. - let key = if let Some(cache) = cache { - if let Ok(key) = cache_key(executable) { - if let Ok(data) = cacache::read_sync(cache, &key) { - debug!("Using cached markers for {}", executable.display()); - return Ok(serde_json::from_slice::(&data)?); - } - Some(key) - } else { - None - } - } else { - None - }; - - // Otherwise, run the Python script. - debug!("Detecting markers for {}", executable.display()); - let markers = detect_markers(executable)?; - - // Write to the cache. - if let Some(cache) = cache { - if let Some(key) = key { - cacache::write_sync(cache, key, serde_json::to_vec(&markers)?)?; - } - } - - Ok(markers) -} - -/// Create a cache key for the Python executable, consisting of the executable's -/// last modified time and the executable's path. -fn cache_key(executable: &Path) -> Result { - let modified = executable - .metadata()? - .modified()? - .duration_since(std::time::UNIX_EPOCH)? - .as_millis(); - Ok(format!("puffin:v0:{}:{}", executable.display(), modified)) -} - -const CAPTURE_MARKERS_SCRIPT: &str = " -import os -import sys -import platform -import json -def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) - kind = info.releaselevel - if kind != 'final': - version += kind[0] + str(info.serial) - return version - -if hasattr(sys, 'implementation'): - implementation_version = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name -else: - implementation_version = '0' - implementation_name = '' -bindings = { - 'implementation_name': implementation_name, - 'implementation_version': implementation_version, - 'os_name': os.name, - 'platform_machine': platform.machine(), - 'platform_python_implementation': platform.python_implementation(), - 'platform_release': platform.release(), - 'platform_system': platform.system(), - 'platform_version': platform.version(), - 'python_full_version': platform.python_version(), - 'python_version': '.'.join(platform.python_version_tuple()[:2]), - 'sys_platform': sys.platform, -} -json.dump(bindings, sys.stdout) -sys.stdout.flush() -"; - -/// Run a Python script and return its output. -fn call_python(python: &Path, args: I) -> Result -where - I: IntoIterator, - S: AsRef, -{ - Command::new(python) - .args(args) - .output() - .context(format!("Failed to run `python` at: {:?}", &python)) -} diff --git a/crates/puffin-interpreter/src/python_platform.rs b/crates/puffin-interpreter/src/python_platform.rs index c6245ac66..a209edf3f 100644 --- a/crates/puffin-interpreter/src/python_platform.rs +++ b/crates/puffin-interpreter/src/python_platform.rs @@ -6,22 +6,22 @@ use platform_host::{Os, Platform}; /// A Python-aware wrapper around [`Platform`]. #[derive(Debug, Clone, Eq, PartialEq)] -pub(crate) struct PythonPlatform(Platform); +pub(crate) struct PythonPlatform(pub(crate) Platform); impl PythonPlatform { /// Returns the path to the `python` executable inside a virtual environment. - pub(crate) fn venv_python(&self, venv_base: impl AsRef) -> PathBuf { + pub(crate) fn venv_python(&self, venv_root: impl AsRef) -> PathBuf { let python = if matches!(self.0.os(), Os::Windows) { "python.exe" } else { "python" }; - self.venv_bin_dir(venv_base).join(python) + self.venv_bin_dir(venv_root).join(python) } /// Returns the directory in which the binaries are stored inside a virtual environment. - pub(crate) fn venv_bin_dir(&self, venv_base: impl AsRef) -> PathBuf { - let venv = venv_base.as_ref(); + pub(crate) fn venv_bin_dir(&self, venv_root: impl AsRef) -> PathBuf { + let venv = venv_root.as_ref(); if matches!(self.0.os(), Os::Windows) { let bin_dir = venv.join("Scripts"); if bin_dir.join("python.exe").exists() { @@ -43,10 +43,10 @@ impl PythonPlatform { /// Returns the path to the `site-packages` directory inside a virtual environment. pub(crate) fn venv_site_packages( &self, - venv_base: impl AsRef, + venv_root: impl AsRef, version: (u8, u8), ) -> PathBuf { - let venv = venv_base.as_ref(); + let venv = venv_root.as_ref(); if matches!(self.0.os(), Os::Windows) { venv.join("Lib").join("site-packages") } else { diff --git a/crates/puffin-interpreter/src/virtual_env.rs b/crates/puffin-interpreter/src/virtual_env.rs index 4e842461d..6352ec3e3 100644 --- a/crates/puffin-interpreter/src/virtual_env.rs +++ b/crates/puffin-interpreter/src/virtual_env.rs @@ -1,11 +1,91 @@ use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use crate::InterpreterInfo; use anyhow::{bail, Result}; +use platform_host::Platform; use tracing::debug; use crate::python_platform::PythonPlatform; +/// A Python executable and its associated platform markers. +#[derive(Debug, Clone)] +pub struct Virtualenv { + root: PathBuf, + interpreter_info: InterpreterInfo, +} + +impl Virtualenv { + /// Venv the current Python executable from the host environment. + pub fn from_env(platform: Platform, cache: Option<&Path>) -> Result { + let platform = PythonPlatform::from(platform); + let venv = detect_virtual_env(&platform)?; + let executable = platform.venv_python(&venv); + let interpreter_info = InterpreterInfo::query_cached(&executable, platform.0, cache)?; + + Ok(Self { + root: venv, + interpreter_info, + }) + } + + pub fn from_virtualenv(platform: Platform, root: &Path, cache: Option<&Path>) -> Result { + let platform = PythonPlatform::from(platform); + let executable = platform.venv_python(root); + let interpreter_info = InterpreterInfo::query_cached(&executable, platform.0, cache)?; + + Ok(Self { + root: root.to_path_buf(), + interpreter_info, + }) + } + + /// Creating a new venv from a python interpreter changes this + pub fn new_prefix(venv: &Path, interpreter_info: &InterpreterInfo) -> Self { + Self { + root: venv.to_path_buf(), + interpreter_info: InterpreterInfo { + base_prefix: venv.to_path_buf(), + ..interpreter_info.clone() + }, + } + } + + /// Returns the location of the python interpreter + pub fn python_executable(&self) -> PathBuf { + #[cfg(unix)] + { + self.root.join("bin").join("python") + } + #[cfg(windows)] + { + self.0 + .join("Scripts") + .join("python.exe") + .into_std_path_buf() + } + #[cfg(not(any(unix, windows)))] + { + compile_error!("Only windows and unix (linux, mac os, etc.) are supported") + } + } + + pub fn root(&self) -> &Path { + &self.root + } + + pub fn interpreter_info(&self) -> &InterpreterInfo { + &self.interpreter_info + } + + /// Returns the path to the `site-packages` directory inside a virtual environment. + pub fn site_packages(&self) -> PathBuf { + self.interpreter_info + .platform + .venv_site_packages(&self.root, self.interpreter_info().simple_version()) + } +} + /// Locate the current virtual environment. pub(crate) fn detect_virtual_env(target: &PythonPlatform) -> Result { match (env::var_os("VIRTUAL_ENV"), env::var_os("CONDA_PREFIX")) { diff --git a/crates/puffin-resolver/Cargo.toml b/crates/puffin-resolver/Cargo.toml index 5f11c37c7..79348b151 100644 --- a/crates/puffin-resolver/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -10,7 +10,6 @@ authors = { workspace = true } license = { workspace = true } [dependencies] -gourgeist = { path = "../gourgeist" } install-wheel-rs = { path = "../install-wheel-rs" } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } @@ -39,10 +38,10 @@ tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } url = { workspace = true } waitmap = { workspace = true } -which = { workspace = true } zip = { workspace = true } [dev-dependencies] +gourgeist = { path = "../gourgeist" } puffin-interpreter = { path = "../puffin-interpreter" } once_cell = { version = "1.18.0" } diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index a5aea98df..b8f4be5c1 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -9,14 +9,13 @@ use std::pin::Pin; use std::str::FromStr; use anyhow::Result; -use gourgeist::Venv; use once_cell::sync::Lazy; use pep508_rs::{MarkerEnvironment, Requirement, StringVersion}; use platform_host::{Arch, Os, Platform}; use platform_tags::Tags; use puffin_client::RegistryClientBuilder; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::{InterpreterInfo, Virtualenv}; use puffin_resolver::{ResolutionMode, Resolver}; use puffin_traits::BuildContext; @@ -27,7 +26,11 @@ impl BuildContext for DummyContext { panic!("The test should not need to build source distributions") } - fn python(&self) -> &PythonExecutable { + fn interpreter_info(&self) -> &InterpreterInfo { + panic!("The test should not need to build source distributions") + } + + fn base_python(&self) -> &Path { panic!("The test should not need to build source distributions") } @@ -41,7 +44,7 @@ impl BuildContext for DummyContext { fn install<'a>( &'a self, _requirements: &'a [Requirement], - _venv: &'a Venv, + _venv: &'a Virtualenv, ) -> Pin> + 'a>> { panic!("The test should not need to build source distributions") } diff --git a/crates/puffin-traits/Cargo.toml b/crates/puffin-traits/Cargo.toml index fd5331fd2..d297ab666 100644 --- a/crates/puffin-traits/Cargo.toml +++ b/crates/puffin-traits/Cargo.toml @@ -10,7 +10,6 @@ authors = { workspace = true } license = { workspace = true } [dependencies] -gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } puffin-interpreter = { path = "../puffin-interpreter" } diff --git a/crates/puffin-traits/src/lib.rs b/crates/puffin-traits/src/lib.rs index 898a1e8ce..1abe1aed4 100644 --- a/crates/puffin-traits/src/lib.rs +++ b/crates/puffin-traits/src/lib.rs @@ -4,9 +4,8 @@ use std::future::Future; use std::path::Path; use std::pin::Pin; -use gourgeist::Venv; use pep508_rs::Requirement; -use puffin_interpreter::PythonExecutable; +use puffin_interpreter::{InterpreterInfo, Virtualenv}; /// Avoid cyclic crate dependencies between resolver, installer and builder. /// @@ -53,7 +52,9 @@ pub trait BuildContext { /// All (potentially nested) source distribution builds use the same base python and can reuse /// it's metadata (e.g. wheel compatibility tags). - fn python(&self) -> &PythonExecutable; + fn interpreter_info(&self) -> &InterpreterInfo; + /// The system (or conda) python interpreter to create venvs. + fn base_python(&self) -> &Path; /// Resolve the given requirements into a ready-to-install set of package versions. fn resolve<'a>( @@ -65,7 +66,7 @@ pub trait BuildContext { fn install<'a>( &'a self, requirements: &'a [Requirement], - venv: &'a Venv, + venv: &'a Virtualenv, ) -> Pin> + 'a>>; /// Build a source distribution into a wheel from an archive. /// diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..3a6f29ca8 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,4 @@ +exclude = [ + "crates/gourgeist/src/activator/activate_this.py", + "crates/gourgeist/src/_virtualenv.py" +] \ No newline at end of file