diff --git a/Cargo.lock b/Cargo.lock index 738c54ea1..3aac18739 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -753,15 +753,6 @@ dependencies = [ "dirs-sys", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-sys" version = "0.4.1" @@ -1094,18 +1085,11 @@ version = "0.0.4" dependencies = [ "camino", "clap", - "configparser", - "dirs", - "distribution-filename", "fs-err", - "install-wheel-rs", "platform-host", "puffin-interpreter", - "rayon", - "reqwest", "serde", "serde_json", - "tempfile", "thiserror", "tracing", "tracing-subscriber", @@ -2168,14 +2152,12 @@ name = "puffin-build" version = "0.0.1" dependencies = [ "anyhow", - "clap", "flate2", "fs-err", "gourgeist", "indoc", "pep508_rs", "platform-host", - "platform-tags", "puffin-interpreter", "puffin-traits", "pyproject-toml", diff --git a/crates/gourgeist/Cargo.toml b/crates/gourgeist/Cargo.toml index 280b3c47e..b51d10925 100644 --- a/crates/gourgeist/Cargo.toml +++ b/crates/gourgeist/Cargo.toml @@ -14,27 +14,15 @@ authors = { workspace = true } license = { workspace = true } [dependencies] -distribution-filename = { path = "../distribution-filename" } -install-wheel-rs = { path = "../install-wheel-rs", optional = true } platform-host = { path = "../platform-host" } puffin-interpreter = { path = "../puffin-interpreter" } camino = { workspace = true } clap = { workspace = true } -configparser = { workspace = true } -dirs = { workspace = true } fs-err = { workspace = true } -reqwest = { workspace = true, optional = true, default-features = false, features = ["blocking"] } -rayon = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } -tempfile = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } which = { workspace = true } - -[features] -default = ["install"] -install = ["install-wheel-rs", "reqwest"] -parallel = ["rayon"] diff --git a/crates/gourgeist/src/lib.rs b/crates/gourgeist/src/lib.rs index 6f2905763..c975c710e 100644 --- a/crates/gourgeist/src/lib.rs +++ b/crates/gourgeist/src/lib.rs @@ -2,8 +2,6 @@ use std::io; use std::path::{Path, PathBuf}; use camino::{Utf8Path, Utf8PathBuf}; -use dirs::cache_dir; -use tempfile::PersistError; use thiserror::Error; pub use interpreter::parse_python_cli; @@ -14,84 +12,28 @@ use crate::bare::create_bare_venv; mod bare; mod interpreter; -#[cfg(feature = "install")] -mod packages; -#[cfg(not(feature = "install"))] -mod virtualenv_cache; #[derive(Debug, Error)] pub enum Error { #[error(transparent)] IO(#[from] io::Error), - /// It's effectively an io error with extra info - #[error(transparent)] - Persist(#[from] PersistError), - /// Adds url and target path to the io error - #[error("Failed to download wheel from {url} to {path}")] - WheelDownload { - url: String, - path: Utf8PathBuf, - #[source] - err: io::Error, - }, #[error("Failed to determine python interpreter to use")] InvalidPythonInterpreter(#[source] Box), - #[error("Failed to query python interpreter at {interpreter}")] - PythonSubcommand { - interpreter: Utf8PathBuf, - #[source] - err: io::Error, - }, - #[cfg(feature = "install")] - #[error("Failed to contact pypi")] - Request(#[from] reqwest::Error), - #[cfg(feature = "install")] - #[error("Failed to install {package}")] - InstallWheel { - package: String, - #[source] - err: install_wheel_rs::Error, - }, #[error("{0} is not a valid UTF-8 path")] NonUTF8Path(PathBuf), #[error(transparent)] Platform(#[from] PlatformError), } -pub(crate) fn crate_cache_dir() -> io::Result { - Ok(cache_dir() - .and_then(|path| Utf8PathBuf::from_path_buf(path).ok()) - .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Couldn't detect cache dir"))? - .join(env!("CARGO_PKG_NAME"))) -} - -/// Create a virtualenv and if not bare, install `wheel`, `pip` and `setuptools`. +/// Create a virtualenv. pub fn create_venv( location: impl Into, base_python: impl AsRef, info: &InterpreterInfo, - bare: bool, ) -> Result { let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?; let base_python = Utf8Path::from_path(base_python.as_ref()) .ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?; - let paths = create_bare_venv(&location, base_python, info)?; - - if !bare { - #[cfg(feature = "install")] - { - packages::install_base_packages(&location, info, &paths)?; - } - #[cfg(not(feature = "install"))] - { - virtualenv_cache::install_base_packages( - &paths.bin, - &paths.interpreter, - &paths.site_packages, - )?; - } - } - - Ok(Virtualenv::new_prefix(location.as_std_path(), info)) + Ok(Virtualenv::new_prefix(paths.root.as_std_path(), info)) } diff --git a/crates/gourgeist/src/main.rs b/crates/gourgeist/src/main.rs index 72618d08e..361fd9cce 100644 --- a/crates/gourgeist/src/main.rs +++ b/crates/gourgeist/src/main.rs @@ -17,8 +17,6 @@ struct Cli { path: Option, #[clap(short, long)] python: Option, - #[clap(long)] - bare: bool, } fn run() -> Result<(), gourgeist::Error> { @@ -27,7 +25,7 @@ fn run() -> Result<(), gourgeist::Error> { let python = parse_python_cli(cli.python)?; let platform = Platform::current()?; let info = InterpreterInfo::query(python.as_std_path(), platform, None).unwrap(); - create_venv(location, &python, &info, cli.bare)?; + create_venv(location, &python, &info)?; Ok(()) } diff --git a/crates/gourgeist/src/packages.rs b/crates/gourgeist/src/packages.rs deleted file mode 100644 index 75db4eb03..000000000 --- a/crates/gourgeist/src/packages.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::io; -use std::io::BufWriter; -use std::str::FromStr; - -use camino::{Utf8Path, Utf8PathBuf}; -use fs_err as fs; -use fs_err::File; -#[cfg(feature = "parallel")] -use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use tempfile::NamedTempFile; -use tracing::debug; - -use distribution_filename::WheelFilename; -use install_wheel_rs::{install_wheel, InstallLocation}; -use puffin_interpreter::InterpreterInfo; - -use crate::bare::VenvPaths; -use crate::{crate_cache_dir, Error}; - -pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result { - let wheels_cache = crate_cache_dir()?.join("wheels"); - let cached_wheel = wheels_cache.join(filename); - if cached_wheel.is_file() { - debug!("Using cached wheel at {cached_wheel}"); - return Ok(cached_wheel); - } - - debug!("Downloading wheel from {url} to {cached_wheel}"); - fs::create_dir_all(&wheels_cache)?; - let mut tempfile = NamedTempFile::new_in(wheels_cache)?; - let tempfile_path: Utf8PathBuf = tempfile - .path() - .to_path_buf() - .try_into() - .map_err(camino::FromPathBufError::into_io_error)?; - let mut response = reqwest::blocking::get(url)?; - io::copy(&mut response, &mut BufWriter::new(&mut tempfile)).map_err(|err| { - Error::WheelDownload { - url: url.to_string(), - path: tempfile_path.to_path_buf(), - err, - } - })?; - tempfile.persist(&cached_wheel)?; - Ok(cached_wheel) -} - -/// Install pip, setuptools and wheel from cache pypi with atm fixed wheels -pub(crate) fn install_base_packages( - location: &Utf8Path, - info: &InterpreterInfo, - paths: &VenvPaths, -) -> Result<(), Error> { - let install_location = InstallLocation::new(location.canonicalize()?, info.simple_version()); - let install_location = install_location.acquire_lock()?; - - // TODO(konstin): Use the json api instead - // TODO(konstin): Only check the json API so often (monthly? daily?) - let packages = [ - ("pip-23.2.1-py3-none-any.whl", "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl"), - ("setuptools-68.2.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/bb/26/7945080113158354380a12ce26873dd6c1ebd88d47f5bc24e2c5bb38c16a/setuptools-68.2.2-py3-none-any.whl"), - ("wheel-0.41.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl"), - ]; - #[cfg(feature = "rayon")] - let iterator = packages.into_par_iter(); - #[cfg(not(feature = "rayon"))] - let iterator = packages.into_iter(); - iterator - .map(|(filename, url)| { - let wheel_file = download_wheel_cached(filename, url)?; - let parsed_filename = WheelFilename::from_str(filename).unwrap(); - install_wheel( - &install_location, - File::open(wheel_file)?, - &parsed_filename, - None, - false, - false, - &[], - paths.interpreter.as_std_path(), - ) - .map_err(|err| Error::InstallWheel { - package: filename.to_string(), - err, - })?; - Ok(()) - }) - .collect::, Error>>()?; - Ok(()) -} diff --git a/crates/gourgeist/src/virtualenv_cache.rs b/crates/gourgeist/src/virtualenv_cache.rs deleted file mode 100644 index f73e0104e..000000000 --- a/crates/gourgeist/src/virtualenv_cache.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Deprecated, use only as template when implementing caching - -use std::io; -use std::path::Path; - -use camino::{Utf8Path, Utf8PathBuf}; -use dirs::data_dir; -use fs_err as fs; -use tracing::debug; - -use crate::Error; - -/// Install wheel, pip and setuptools from the cache -pub(crate) fn install_base_packages( - bin_dir: &Utf8Path, - venv_python: &Utf8Path, - site_packages: &Utf8Path, -) -> Result<(), Error> { - // Install packages - // TODO: Implement our own logic: - // * Our own cache and logic to detect whether a wheel is present - // * Check if the version is recent (e.g. update if older than 1 month) - // * Query pypi API if no, parse versions (pep440) and their metadata - // * Download compatible wheel (py3-none-any should do) - // * Install into the cache directory - let prefix = "virtualenv/wheel/3.11/image/1/CopyPipInstall/"; - let wheel_tag = "py3-none-any"; - let packages = &[ - ("pip", "23.2.1"), - ("setuptools", "68.2.2"), - ("wheel", "0.41.2"), - ]; - let virtualenv_data_dir: Utf8PathBuf = data_dir().unwrap().try_into().unwrap(); - for (name, version) in packages { - // TODO: acquire lock - let unpacked_wheel = virtualenv_data_dir - .join(prefix) - .join(format!("{name}-{version}-{wheel_tag}")); - debug!("Installing {name} by copying from {unpacked_wheel}"); - copy_dir_all(&unpacked_wheel, site_packages.as_std_path())?; - - // Generate launcher - // virtualenv for some reason creates extra entrypoints that we don't - // https://github.com/pypa/virtualenv/blob/025e96fbad37f85617364002ae2a0064b09fc984/src/virtualenv/seed/embed/via_app_data/pip_install/base.py#L74-L95 - let ini_text = fs::read_to_string( - site_packages - .join(format!("{name}-{version}.dist-info")) - .join("entry_points.txt"), - )?; - let entry_points_mapping = configparser::ini::Ini::new_cs() - .read(ini_text) - .map_err(|err| format!("{name} entry_points.txt is invalid: {}", err)) - .unwrap(); - for (key, value) in entry_points_mapping - .get("console_scripts") - .cloned() - .unwrap_or_default() - { - let (import_from, function) = value - .as_ref() - .and_then(|value| value.split_once(':')) - .ok_or_else(|| { - format!("{name} entry_points.txt {key} has an invalid value {value:?}") - }) - .unwrap(); - let launcher = bin_dir.join(key); - let launcher_script = unix_launcher_script(venv_python, import_from, function); - fs::write(&launcher, launcher_script)?; - // We need to make the launcher executable - #[cfg(target_family = "unix")] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(launcher, std::fs::Permissions::from_mode(0o755))?; - } - } - } - Ok(()) -} - -/// https://stackoverflow.com/a/65192210/3549270 -pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { - fs::create_dir_all(&dst)?; - for entry in fs::read_dir(src.as_ref())? { - let entry = entry?; - let ty = entry.file_type()?; - if ty.is_dir() { - copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; - } else { - fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; - } - } - Ok(()) -} - -/// Template for the console scripts in the `bin` directory -pub fn unix_launcher_script(python: &Utf8Path, import_from: &str, function: &str) -> String { - format!( - r#"#!{python} - # -*- coding: utf-8 -*- -import re -import sys -from {import_from} import {function} -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit({function}()) -"#, - python = python, - import_from = import_from, - function = function - ) -} diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index 509f8310d..ad0ebcdeb 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -14,12 +14,10 @@ license = { workspace = true } gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } -platform-tags = { path = "../platform-tags" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-traits = { path = "../puffin-traits" } anyhow = { workspace = true } -clap = { workspace = true, features = ["derive"] } flate2 = { workspace = true } fs-err = { workspace = true } indoc = { workspace = true } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index 776d058ab..5fbb38827 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -166,7 +166,6 @@ impl SourceDistributionBuild { temp_dir.path().join(".venv"), build_context.base_python(), interpreter_info, - true, )?; // There are packages such as DTLSSocket 0.1.16 that say diff --git a/crates/puffin-cli/src/commands/venv.rs b/crates/puffin-cli/src/commands/venv.rs index 1bfe7a2c2..080db9033 100644 --- a/crates/puffin-cli/src/commands/venv.rs +++ b/crates/puffin-cli/src/commands/venv.rs @@ -87,7 +87,7 @@ fn venv_impl( .into_diagnostic()?; // Create the virtual environment. - gourgeist::create_venv(path, &base_python, &interpreter_info, true) + gourgeist::create_venv(path, &base_python, &interpreter_info) .map_err(VenvError::CreationError)?; Ok(ExitStatus::Success)