Remove virtualenv setup from gourgeist (#339)

We now only support building bare environments.

Cargo.lock

@@ -753,15 +753,6 @@ dependencies = [
  "dirs-sys",
 ]
 
-[[package]]
-name = "dirs"
-version = "5.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
-dependencies = [
- "dirs-sys",
-]
-
 [[package]]
 name = "dirs-sys"
 version = "0.4.1"
@@ -1094,18 +1085,11 @@ version = "0.0.4"
 dependencies = [
  "camino",
  "clap",
- "configparser",
- "dirs",
- "distribution-filename",
  "fs-err",
- "install-wheel-rs",
  "platform-host",
  "puffin-interpreter",
- "rayon",
- "reqwest",
  "serde",
  "serde_json",
- "tempfile",
  "thiserror",
  "tracing",
  "tracing-subscriber",
@@ -2168,14 +2152,12 @@ name = "puffin-build"
 version = "0.0.1"
 dependencies = [
  "anyhow",
- "clap",
  "flate2",
  "fs-err",
  "gourgeist",
  "indoc",
  "pep508_rs",
  "platform-host",
- "platform-tags",
  "puffin-interpreter",
  "puffin-traits",
  "pyproject-toml",

crates/gourgeist/Cargo.toml

@@ -14,27 +14,15 @@ authors = { workspace = true }
 license = { workspace = true }
 
 [dependencies]
-distribution-filename = { path = "../distribution-filename" }
-install-wheel-rs = { path = "../install-wheel-rs", optional = true }
 platform-host = { path = "../platform-host" }
 puffin-interpreter = { path = "../puffin-interpreter" }
 
 camino = { workspace = true }
 clap = { workspace = true }
-configparser = { workspace = true }
-dirs = { workspace = true }
 fs-err = { workspace = true }
-reqwest = { workspace = true, optional = true, default-features = false, features = ["blocking"] }
-rayon = { workspace = true, optional = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-tempfile = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 which = { workspace = true }
-
-[features]
-default = ["install"]
-install = ["install-wheel-rs", "reqwest"]
-parallel = ["rayon"]

crates/gourgeist/src/lib.rs

@@ -2,8 +2,6 @@ use std::io;
 use std::path::{Path, PathBuf};
 
 use camino::{Utf8Path, Utf8PathBuf};
-use dirs::cache_dir;
-use tempfile::PersistError;
 use thiserror::Error;
 
 pub use interpreter::parse_python_cli;
@@ -14,84 +12,28 @@ use crate::bare::create_bare_venv;
 
 mod bare;
 mod interpreter;
-#[cfg(feature = "install")]
-mod packages;
-#[cfg(not(feature = "install"))]
-mod virtualenv_cache;
 
 #[derive(Debug, Error)]
 pub enum Error {
     #[error(transparent)]
     IO(#[from] io::Error),
-    /// It's effectively an io error with extra info
-    #[error(transparent)]
-    Persist(#[from] PersistError),
-    /// Adds url and target path to the io error
-    #[error("Failed to download wheel from {url} to {path}")]
-    WheelDownload {
-        url: String,
-        path: Utf8PathBuf,
-        #[source]
-        err: io::Error,
-    },
     #[error("Failed to determine python interpreter to use")]
     InvalidPythonInterpreter(#[source] Box<dyn std::error::Error + Sync + Send>),
-    #[error("Failed to query python interpreter at {interpreter}")]
-    PythonSubcommand {
-        interpreter: Utf8PathBuf,
-        #[source]
-        err: io::Error,
-    },
-    #[cfg(feature = "install")]
-    #[error("Failed to contact pypi")]
-    Request(#[from] reqwest::Error),
-    #[cfg(feature = "install")]
-    #[error("Failed to install {package}")]
-    InstallWheel {
-        package: String,
-        #[source]
-        err: install_wheel_rs::Error,
-    },
     #[error("{0} is not a valid UTF-8 path")]
     NonUTF8Path(PathBuf),
     #[error(transparent)]
     Platform(#[from] PlatformError),
 }
 
-pub(crate) fn crate_cache_dir() -> io::Result<Utf8PathBuf> {
+/// Create a virtualenv.
-    Ok(cache_dir()
-        .and_then(|path| Utf8PathBuf::from_path_buf(path).ok())
-        .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Couldn't detect cache dir"))?
-        .join(env!("CARGO_PKG_NAME")))
-}
-
-/// Create a virtualenv and if not bare, install `wheel`, `pip` and `setuptools`.
 pub fn create_venv(
     location: impl Into<PathBuf>,
     base_python: impl AsRef<Path>,
     info: &InterpreterInfo,
-    bare: bool,
 ) -> Result<Virtualenv, Error> {
     let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?;
     let base_python = Utf8Path::from_path(base_python.as_ref())
         .ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?;
-
     let paths = create_bare_venv(&location, base_python, info)?;
-
+    Ok(Virtualenv::new_prefix(paths.root.as_std_path(), info))
-    if !bare {
-        #[cfg(feature = "install")]
-        {
-            packages::install_base_packages(&location, info, &paths)?;
-        }
-        #[cfg(not(feature = "install"))]
-        {
-            virtualenv_cache::install_base_packages(
-                &paths.bin,
-                &paths.interpreter,
-                &paths.site_packages,
-            )?;
-        }
-    }
-
-    Ok(Virtualenv::new_prefix(location.as_std_path(), info))
 }

crates/gourgeist/src/main.rs

@@ -17,8 +17,6 @@ struct Cli {
     path: Option<Utf8PathBuf>,
     #[clap(short, long)]
     python: Option<Utf8PathBuf>,
-    #[clap(long)]
-    bare: bool,
 }
 
 fn run() -> Result<(), gourgeist::Error> {
@@ -27,7 +25,7 @@ fn run() -> Result<(), gourgeist::Error> {
     let python = parse_python_cli(cli.python)?;
     let platform = Platform::current()?;
     let info = InterpreterInfo::query(python.as_std_path(), platform, None).unwrap();
-    create_venv(location, &python, &info, cli.bare)?;
+    create_venv(location, &python, &info)?;
 
     Ok(())
 }

crates/gourgeist/src/packages.rs

@@ -1,90 +0,0 @@
-use std::io;
-use std::io::BufWriter;
-use std::str::FromStr;
-
-use camino::{Utf8Path, Utf8PathBuf};
-use fs_err as fs;
-use fs_err::File;
-#[cfg(feature = "parallel")]
-use rayon::iter::{IntoParallelIterator, ParallelIterator};
-use tempfile::NamedTempFile;
-use tracing::debug;
-
-use distribution_filename::WheelFilename;
-use install_wheel_rs::{install_wheel, InstallLocation};
-use puffin_interpreter::InterpreterInfo;
-
-use crate::bare::VenvPaths;
-use crate::{crate_cache_dir, Error};
-
-pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
-    let wheels_cache = crate_cache_dir()?.join("wheels");
-    let cached_wheel = wheels_cache.join(filename);
-    if cached_wheel.is_file() {
-        debug!("Using cached wheel at {cached_wheel}");
-        return Ok(cached_wheel);
-    }
-
-    debug!("Downloading wheel from {url} to {cached_wheel}");
-    fs::create_dir_all(&wheels_cache)?;
-    let mut tempfile = NamedTempFile::new_in(wheels_cache)?;
-    let tempfile_path: Utf8PathBuf = tempfile
-        .path()
-        .to_path_buf()
-        .try_into()
-        .map_err(camino::FromPathBufError::into_io_error)?;
-    let mut response = reqwest::blocking::get(url)?;
-    io::copy(&mut response, &mut BufWriter::new(&mut tempfile)).map_err(|err| {
-        Error::WheelDownload {
-            url: url.to_string(),
-            path: tempfile_path.to_path_buf(),
-            err,
-        }
-    })?;
-    tempfile.persist(&cached_wheel)?;
-    Ok(cached_wheel)
-}
-
-/// Install pip, setuptools and wheel from cache pypi with atm fixed wheels
-pub(crate) fn install_base_packages(
-    location: &Utf8Path,
-    info: &InterpreterInfo,
-    paths: &VenvPaths,
-) -> Result<(), Error> {
-    let install_location = InstallLocation::new(location.canonicalize()?, info.simple_version());
-    let install_location = install_location.acquire_lock()?;
-
-    // TODO(konstin): Use the json api instead
-    // TODO(konstin): Only check the json API so often (monthly? daily?)
-    let packages = [
-        ("pip-23.2.1-py3-none-any.whl", "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl"),
-        ("setuptools-68.2.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/bb/26/7945080113158354380a12ce26873dd6c1ebd88d47f5bc24e2c5bb38c16a/setuptools-68.2.2-py3-none-any.whl"),
-        ("wheel-0.41.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl"),
-    ];
-    #[cfg(feature = "rayon")]
-    let iterator = packages.into_par_iter();
-    #[cfg(not(feature = "rayon"))]
-    let iterator = packages.into_iter();
-    iterator
-        .map(|(filename, url)| {
-            let wheel_file = download_wheel_cached(filename, url)?;
-            let parsed_filename = WheelFilename::from_str(filename).unwrap();
-            install_wheel(
-                &install_location,
-                File::open(wheel_file)?,
-                &parsed_filename,
-                None,
-                false,
-                false,
-                &[],
-                paths.interpreter.as_std_path(),
-            )
-            .map_err(|err| Error::InstallWheel {
-                package: filename.to_string(),
-                err,
-            })?;
-            Ok(())
-        })
-        .collect::<Result<Vec<()>, Error>>()?;
-    Ok(())
-}

crates/gourgeist/src/virtualenv_cache.rs

@@ -1,111 +0,0 @@
-//! Deprecated, use only as template when implementing caching
-
-use std::io;
-use std::path::Path;
-
-use camino::{Utf8Path, Utf8PathBuf};
-use dirs::data_dir;
-use fs_err as fs;
-use tracing::debug;
-
-use crate::Error;
-
-/// Install wheel, pip and setuptools from the cache
-pub(crate) fn install_base_packages(
-    bin_dir: &Utf8Path,
-    venv_python: &Utf8Path,
-    site_packages: &Utf8Path,
-) -> Result<(), Error> {
-    // Install packages
-    // TODO: Implement our own logic:
-    //  * Our own cache and logic to detect whether a wheel is present
-    //  * Check if the version is recent (e.g. update if older than 1 month)
-    //  * Query pypi API if no, parse versions (pep440) and their metadata
-    //  * Download compatible wheel (py3-none-any should do)
-    //  * Install into the cache directory
-    let prefix = "virtualenv/wheel/3.11/image/1/CopyPipInstall/";
-    let wheel_tag = "py3-none-any";
-    let packages = &[
-        ("pip", "23.2.1"),
-        ("setuptools", "68.2.2"),
-        ("wheel", "0.41.2"),
-    ];
-    let virtualenv_data_dir: Utf8PathBuf = data_dir().unwrap().try_into().unwrap();
-    for (name, version) in packages {
-        // TODO: acquire lock
-        let unpacked_wheel = virtualenv_data_dir
-            .join(prefix)
-            .join(format!("{name}-{version}-{wheel_tag}"));
-        debug!("Installing {name} by copying from {unpacked_wheel}");
-        copy_dir_all(&unpacked_wheel, site_packages.as_std_path())?;
-
-        // Generate launcher
-        // virtualenv for some reason creates extra entrypoints that we don't
-        // https://github.com/pypa/virtualenv/blob/025e96fbad37f85617364002ae2a0064b09fc984/src/virtualenv/seed/embed/via_app_data/pip_install/base.py#L74-L95
-        let ini_text = fs::read_to_string(
-            site_packages
-                .join(format!("{name}-{version}.dist-info"))
-                .join("entry_points.txt"),
-        )?;
-        let entry_points_mapping = configparser::ini::Ini::new_cs()
-            .read(ini_text)
-            .map_err(|err| format!("{name} entry_points.txt is invalid: {}", err))
-            .unwrap();
-        for (key, value) in entry_points_mapping
-            .get("console_scripts")
-            .cloned()
-            .unwrap_or_default()
-        {
-            let (import_from, function) = value
-                .as_ref()
-                .and_then(|value| value.split_once(':'))
-                .ok_or_else(|| {
-                    format!("{name} entry_points.txt {key} has an invalid value {value:?}")
-                })
-                .unwrap();
-            let launcher = bin_dir.join(key);
-            let launcher_script = unix_launcher_script(venv_python, import_from, function);
-            fs::write(&launcher, launcher_script)?;
-            // We need to make the launcher executable
-            #[cfg(target_family = "unix")]
-            {
-                use std::os::unix::fs::PermissionsExt;
-                fs::set_permissions(launcher, std::fs::Permissions::from_mode(0o755))?;
-            }
-        }
-    }
-    Ok(())
-}
-
-/// https://stackoverflow.com/a/65192210/3549270
-pub fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
-    fs::create_dir_all(&dst)?;
-    for entry in fs::read_dir(src.as_ref())? {
-        let entry = entry?;
-        let ty = entry.file_type()?;
-        if ty.is_dir() {
-            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        } else {
-            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        }
-    }
-    Ok(())
-}
-
-/// Template for the console scripts in the `bin` directory
-pub fn unix_launcher_script(python: &Utf8Path, import_from: &str, function: &str) -> String {
-    format!(
-        r#"#!{python}
-    # -*- coding: utf-8 -*-
-import re
-import sys
-from {import_from} import {function}
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit({function}())
-"#,
-        python = python,
-        import_from = import_from,
-        function = function
-    )
-}

crates/puffin-build/Cargo.toml

@@ -14,12 +14,10 @@ license = { workspace = true }
 gourgeist = { path = "../gourgeist" }
 pep508_rs = { path = "../pep508-rs" }
 platform-host = { path = "../platform-host" }
-platform-tags = { path = "../platform-tags" }
 puffin-interpreter = { path = "../puffin-interpreter" }
 puffin-traits = { path = "../puffin-traits" }
 
 anyhow = { workspace = true }
-clap = { workspace = true, features = ["derive"] }
 flate2 = { workspace = true }
 fs-err = { workspace = true }
 indoc = { workspace = true }

crates/puffin-build/src/lib.rs

@@ -166,7 +166,6 @@ impl SourceDistributionBuild {
             temp_dir.path().join(".venv"),
             build_context.base_python(),
             interpreter_info,
-            true,
         )?;
 
         // There are packages such as DTLSSocket 0.1.16 that say

crates/puffin-cli/src/commands/venv.rs

@@ -87,7 +87,7 @@ fn venv_impl(
     .into_diagnostic()?;
 
     // Create the virtual environment.
-    gourgeist::create_venv(path, &base_python, &interpreter_info, true)
+    gourgeist::create_venv(path, &base_python, &interpreter_info)
         .map_err(VenvError::CreationError)?;
 
     Ok(ExitStatus::Success)