Use local copy of gourgeist (#62)

This PR gets `gourgeist` passing our local CI and integrated into the
broader workspace.

There's some duplicate between concepts in `gourgeist` (like the
`InterpreterInfo`) and structs we have elsewhere, but we can tackle
those later.

Cargo.lock

@@ -287,6 +287,15 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "camino"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "cast"
 version = "0.3.0"
@@ -607,6 +616,15 @@ dependencies = [
  "dirs-sys",
 ]
 
+[[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
 [[package]]
 name = "dirs-sys"
 version = "0.4.1"
@@ -642,25 +660,14 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480"
+checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
 dependencies = [
- "errno-dragonfly",
  "libc",
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
 [[package]]
 name = "fastrand"
 version = "2.0.1"
@@ -861,6 +868,29 @@ dependencies = [
  "scroll",
 ]
 
+[[package]]
+name = "gourgeist"
+version = "0.0.4"
+dependencies = [
+ "camino",
+ "clap",
+ "configparser",
+ "dirs",
+ "fs-err",
+ "install-wheel-rs",
+ "rayon",
+ "reqwest",
+ "seahash",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "thiserror",
+ "tracing",
+ "tracing-subscriber",
+ "wheel-filename",
+ "which",
+]
+
 [[package]]
 name = "h2"
 version = "0.3.21"
@@ -913,6 +943,15 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "home"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
+dependencies = [
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -1211,9 +1250,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.148"
+version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
 
 [[package]]
 name = "line-wrap"
@@ -1232,9 +1271,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db"
+checksum = "45786cec4d5e54a224b15cb9f06751883103a27c19c93eda09b0b4f5f08fefac"
 
 [[package]]
 name = "lock_api"
@@ -1385,9 +1424,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
+checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
 dependencies = [
  "autocfg",
 ]
@@ -1487,7 +1526,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
 dependencies = [
  "instant",
  "lock_api",
- "parking_lot_core",
+ "parking_lot_core 0.8.6",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core 0.9.8",
 ]
 
 [[package]]
@@ -1504,6 +1553,19 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "parking_lot_core"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall 0.3.5",
+ "smallvec",
+ "windows-targets 0.48.5",
+]
+
 [[package]]
 name = "pep440_rs"
 version = "0.3.12"
@@ -1793,7 +1855,7 @@ dependencies = [
  "indoc 1.0.9",
  "libc",
  "memoffset",
- "parking_lot",
+ "parking_lot 0.12.1",
  "pyo3-build-config",
  "pyo3-ffi",
  "pyo3-macros",
@@ -2082,7 +2144,7 @@ dependencies = [
  "getrandom",
  "http",
  "hyper",
- "parking_lot",
+ "parking_lot 0.11.2",
  "reqwest",
  "reqwest-middleware",
  "retry-policies",
@@ -2192,6 +2254,12 @@ dependencies = [
  "syn 2.0.38",
 ]
 
+[[package]]
+name = "seahash"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+
 [[package]]
 name = "security-framework"
 version = "2.9.2"
@@ -2302,9 +2370,9 @@ dependencies = [
 
 [[package]]
 name = "similar"
-version = "2.2.1"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf"
+checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597"
 
 [[package]]
 name = "slab"
@@ -2887,7 +2955,7 @@ checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
 dependencies = [
  "futures",
  "js-sys",
- "parking_lot",
+ "parking_lot 0.11.2",
  "pin-utils",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -2912,6 +2980,18 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

Cargo.toml

@@ -15,12 +15,14 @@ license = "MIT OR Apache-2.0"
 anyhow = { version = "1.0.75" }
 bitflags = { version = "2.4.0" }
 cacache = { version = "11.7.1", default-features = false, features = ["tokio-runtime"] }
+camino = { version = "1.1.6", features = ["serde1"] }
 clap = { version = "4.4.6" }
 colored = { version = "2.0.4" }
 configparser = { version = "3.0.2" }
 csv = { version = "1.3.0" }
 data-encoding = { version = "2.4.0" }
 directories = { version = "5.0.1" }
+dirs = { version = "5.0.1" }
 fs-err = { version = "2.9.0" }
 fs2 = { version = "0.4.3" }
 futures = { version = "0.3.28" }
@@ -39,6 +41,7 @@ reqwest = { version = "0.11.22", features = ["json", "gzip", "brotli", "stream"]
 reqwest-middleware = { version = "0.2.3" }
 reqwest-retry = { version = "0.3.0" }
 rfc2047-decoder = { version = "1.0.1" }
+seahash = { version = "4.1.0" }
 serde = { version = "1.0.188" }
 serde_json = { version = "1.0.107" }
 sha2 = { version = "0.10.8" }
@@ -53,4 +56,5 @@ tracing-tree = { version = "0.2.5" }
 unicode-width = { version = "0.1.8" }
 url = { version = "2.4.1" }
 walkdir = { version = "2.4.0" }
+which = { version = "4.4.2" }
 zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

crates/gourgeist/Cargo.toml

@@ -8,54 +8,27 @@ license = "MIT OR Apache-2.0"
 keywords = ["virtualenv", "venv", "python"]
 readme = "Readme.md"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-camino = { version = "1.1.6", features = ["serde1"] }
+install-wheel-rs = { path = "../install-wheel-rs", optional = true }
-clap = { version = "4.4.5", features = ["derive"] }
+wheel-filename = { path = "../wheel-filename" }
-configparser = "3.0.2"
+
-dirs = "5.0.1"
+camino = { workspace = true }
-fs-err = "2.9.0"
+clap = { workspace = true }
-install-wheel-rs = { version = "0.0.1", optional = true }
+configparser = { workspace = true }
-minreq = { version = "2.10.0", optional = true, features = ["https"] }
+dirs = { workspace = true }
-rayon = { version = "1.8.0", optional = true }
+fs-err = { workspace = true }
-seahash = "4.1.0"
+reqwest = { workspace = true, optional = true, features = ["blocking"] }
-serde = { version = "1.0.188", features = ["derive"] }
+rayon = { workspace = true, optional = true }
-serde_json = "1.0.107"
+seahash = { workspace = true }
-tempfile = "3.8.0"
+serde = { workspace = true }
-thiserror = "1.0.49"
+serde_json = { workspace = true }
-tracing = "0.1.37"
+tempfile = { workspace = true }
-tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+thiserror = { workspace = true }
-which = "4.4.2"
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+which = { workspace = true }
 
 [features]
 default = ["install"]
-install = ["install-wheel-rs", "minreq"]
+install = ["install-wheel-rs", "reqwest"]
 parallel = ["rayon"]
-
-# zip implementation
-[profile.dev.package.adler]
-opt-level = 3
-
-[profile.profiling]
-inherits = "release"
-lto = "thin"
-debug = true
-
-# The profile that 'cargo dist' will build with
-[profile.dist]
-inherits = "release"
-lto = "thin"
-
-# Config for 'cargo dist'
-[workspace.metadata.dist]
-# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
-cargo-dist-version = "0.3.1"
-# CI backends to support
-ci = ["github"]
-# The installers to generate for each app
-installers = ["shell", "powershell"]
-# Target platforms to build apps for (Rust target-triple syntax)
-targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"]
-# Publish jobs to run in CI
-pr-run-mode = "plan"

crates/gourgeist/src/bare.rs

@@ -1,15 +1,17 @@
 //! Create a bare virtualenv without any packages install
 
-use crate::interpreter::InterpreterInfo;
+use std::io;
+use std::io::{BufWriter, Write};
+
 use camino::{Utf8Path, Utf8PathBuf};
 use fs_err as fs;
 #[cfg(unix)]
 use fs_err::os::unix::fs::symlink;
 use fs_err::File;
-use std::io;
-use std::io::{BufWriter, Write};
 use tracing::info;
 
+use crate::interpreter::InterpreterInfo;
+
 /// The bash activate scripts with the venv dependent paths patches out
 const ACTIVATE_TEMPLATES: &[(&str, &str)] = &[
     ("activate", include_str!("activator/activate")),
@@ -27,27 +29,34 @@ const VIRTUALENV_PATCH: &str = include_str!("_virtualenv.py");
 /// Very basic `.cfg` file format writer.
 fn write_cfg(f: &mut impl Write, data: &[(&str, String); 8]) -> io::Result<()> {
     for (key, value) in data {
-        writeln!(f, "{} = {}", key, value)?;
+        writeln!(f, "{key} = {value}")?;
     }
     Ok(())
 }
 
 /// Absolute paths of the virtualenv
 #[derive(Debug)]
-pub struct VenvPaths {
+pub(crate) struct VenvPaths {
     /// The location of the virtualenv, e.g. `.venv`
-    pub root: Utf8PathBuf,
+    #[allow(unused)]
+    pub(crate) root: Utf8PathBuf,
+
     /// The python interpreter.rs inside the virtualenv, on unix `.venv/bin/python`
-    pub interpreter: Utf8PathBuf,
+    #[allow(unused)]
+    pub(crate) interpreter: Utf8PathBuf,
+
     /// The directory with the scripts, on unix `.venv/bin`
-    pub bin: Utf8PathBuf,
+    #[allow(unused)]
+    pub(crate) bin: Utf8PathBuf,
+
     /// The site-packages directory where all the packages are installed to, on unix
     /// and python 3.11 `.venv/lib/python3.11/site-packages`
-    pub site_packages: Utf8PathBuf,
+    #[allow(unused)]
+    pub(crate) site_packages: Utf8PathBuf,
 }
 
 /// Write all the files that belong to a venv without any packages installed.
-pub fn create_bare_venv(
+pub(crate) fn create_bare_venv(
     location: &Utf8Path,
     base_python: &Utf8Path,
     info: &InterpreterInfo,

crates/gourgeist/src/interpreter.rs

@@ -1,14 +1,16 @@
-use crate::{crate_cache_dir, Error};
-use camino::{FromPathBufError, Utf8Path, Utf8PathBuf};
-use fs_err as fs;
-use fs_err::File;
-use serde::{Deserialize, Serialize};
 use std::io;
 use std::io::{BufReader, Write};
 use std::process::{Command, Stdio};
 use std::time::SystemTime;
+
+use camino::{Utf8Path, Utf8PathBuf};
+use fs_err as fs;
+use fs_err::File;
+use serde::{Deserialize, Serialize};
 use tracing::{debug, error, info, warn};
 
+use crate::{crate_cache_dir, Error};
+
 const QUERY_PYTHON: &str = include_str!("query_python.py");
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -44,20 +46,19 @@ pub fn get_interpreter_info(interpreter: &Utf8Path) -> Result<InterpreterInfo, E
                 debug!("Using cache entry {cache_file}");
                 if modified == cache_entry.modified && interpreter == cache_entry.interpreter {
                     return Ok(cache_entry.interpreter_info);
-                } else {
+                }
-                    debug!(
+                debug!(
-                        "Removing mismatching cache entry {cache_file} ({} {} {} {})",
+                    "Removing mismatching cache entry {cache_file} ({} {} {} {})",
-                        modified, cache_entry.modified, interpreter, cache_entry.interpreter
+                    modified, cache_entry.modified, interpreter, cache_entry.interpreter
-                    );
+                );
-                    if let Err(remove_err) = fs::remove_file(&cache_file) {
+                if let Err(remove_err) = fs::remove_file(&cache_file) {
-                        warn!("Failed to mismatching cache file at {cache_file}: {remove_err}")
+                    warn!("Failed to mismatching cache file at {cache_file}: {remove_err}");
-                    }
                 }
             }
             Err(cache_err) => {
                 debug!("Removing broken cache entry {cache_file} ({cache_err})");
                 if let Err(remove_err) = fs::remove_file(&cache_file) {
-                    warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})")
+                    warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})");
                 }
             }
         }
@@ -188,7 +189,7 @@ pub fn parse_python_cli(cli_python: Option<Utf8PathBuf>) -> Result<Utf8PathBuf,
                 )
             })?
             .try_into()
-            .map_err(|err: FromPathBufError| err.into_io_error())?;
+            .map_err(camino::FromPathBufError::into_io_error)?;
         info!("Resolved {python} to {python_in_path}");
         python_in_path
     };

crates/gourgeist/src/lib.rs

@@ -1,13 +1,15 @@
-use crate::bare::create_bare_venv;
+use std::io;
+
 use camino::{Utf8Path, Utf8PathBuf};
 use dirs::cache_dir;
-use interpreter::InterpreterInfo;
-use std::io;
 use tempfile::PersistError;
 use thiserror::Error;
 
+use interpreter::InterpreterInfo;
 pub use interpreter::{get_interpreter_info, parse_python_cli};
 
+use crate::bare::create_bare_venv;
+
 mod bare;
 mod interpreter;
 #[cfg(feature = "install")]
@@ -40,7 +42,7 @@ pub enum Error {
     },
     #[cfg(feature = "install")]
     #[error("Failed to contact pypi")]
-    MinReq(#[from] minreq::Error),
+    Request(#[from] reqwest::Error),
     #[cfg(feature = "install")]
     #[error("Failed to install {package}")]
     InstallWheel {

crates/gourgeist/src/main.rs

@@ -1,14 +1,16 @@
-use camino::Utf8PathBuf;
-use clap::Parser;
-use gourgeist::{create_venv, get_interpreter_info, parse_python_cli};
 use std::error::Error;
 use std::process::ExitCode;
 use std::time::Instant;
+
+use camino::Utf8PathBuf;
+use clap::Parser;
 use tracing::info;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
 use tracing_subscriber::{fmt, EnvFilter};
 
+use gourgeist::{create_venv, get_interpreter_info, parse_python_cli};
+
 #[derive(Parser, Debug)]
 struct Cli {
     path: Option<Utf8PathBuf>,
@@ -38,10 +40,17 @@ fn main() -> ExitCode {
     let result = run();
     info!("Took {}ms", start.elapsed().as_millis());
     if let Err(err) = result {
-        eprintln!("💥 virtualenv creator failed");
+        #[allow(clippy::print_stderr)]
+        {
+            eprintln!("💥 virtualenv creator failed");
+        }
+
         let mut last_error: Option<&(dyn Error + 'static)> = Some(&err);
         while let Some(err) = last_error {
-            eprintln!("  Caused by: {}", err);
+            #[allow(clippy::print_stderr)]
+            {
+                eprintln!("  Caused by: {err}");
+            }
             last_error = err.source();
         }
         ExitCode::FAILURE

crates/gourgeist/src/packages.rs

@@ -1,19 +1,23 @@
-use crate::bare::VenvPaths;
-use crate::interpreter::InterpreterInfo;
-use crate::{crate_cache_dir, Error};
-use camino::{FromPathBufError, Utf8Path, Utf8PathBuf};
-use fs_err as fs;
-use fs_err::File;
-use install_wheel_rs::{install_wheel, InstallLocation, WheelFilename};
-#[cfg(feature = "parallel")]
-use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use std::io;
 use std::io::BufWriter;
 use std::str::FromStr;
+
+use camino::{Utf8Path, Utf8PathBuf};
+use fs_err as fs;
+use fs_err::File;
+#[cfg(feature = "parallel")]
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use tempfile::NamedTempFile;
 use tracing::info;
 
-pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
+use install_wheel_rs::{install_wheel, InstallLocation};
+use wheel_filename::WheelFilename;
+
+use crate::bare::VenvPaths;
+use crate::interpreter::InterpreterInfo;
+use crate::{crate_cache_dir, Error};
+
+pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
     let wheels_cache = crate_cache_dir()?.join("wheels");
     let cached_wheel = wheels_cache.join(filename);
     if cached_wheel.is_file() {
@@ -28,8 +32,8 @@ pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, E
         .path()
         .to_path_buf()
         .try_into()
-        .map_err(|err: FromPathBufError| err.into_io_error())?;
+        .map_err(camino::FromPathBufError::into_io_error)?;
-    let mut response = minreq::get(url).send_lazy()?;
+    let mut response = reqwest::blocking::get(url)?;
     io::copy(&mut response, &mut BufWriter::new(&mut tempfile)).map_err(|err| {
         Error::WheelDownload {
             url: url.to_string(),
@@ -42,7 +46,7 @@ pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, E
 }
 
 /// Install pip, setuptools and wheel from cache pypi with atm fixed wheels
-pub fn install_base_packages(
+pub(crate) fn install_base_packages(
     location: &Utf8Path,
     info: &InterpreterInfo,
     paths: &VenvPaths,
@@ -71,7 +75,8 @@ pub fn install_base_packages(
             install_wheel(
                 &install_location,
                 File::open(wheel_file)?,
-                parsed_filename,
+                &parsed_filename,
+                false,
                 false,
                 &[],
                 // Only relevant for monotrail style installation

crates/gourgeist/src/virtualenv_cache.rs

@@ -1,13 +1,15 @@
 //! Deprecated, use only as template when implementing caching
 
-use crate::Error;
+use std::io;
+use std::path::Path;
+
 use camino::{Utf8Path, Utf8PathBuf};
 use dirs::data_dir;
 use fs_err as fs;
-use std::io;
-use std::path::Path;
 use tracing::debug;
 
+use crate::Error;
+
 /// Install wheel, pip and setuptools from the cache
 pub(crate) fn install_base_packages(
     bin_dir: &Utf8Path,