Use local copy of gourgeist (#62)

This PR gets `gourgeist` passing our local CI and integrated into the
broader workspace.

There's some duplicate between concepts in `gourgeist` (like the
`InterpreterInfo`) and structs we have elsewhere, but we can tackle
those later.
This commit is contained in:
Charlie Marsh 2023-10-08 14:45:08 -04:00 committed by GitHub
parent 7caf5f42b8
commit 0ca17a1cf2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 207 additions and 1572 deletions

130
Cargo.lock generated
View file

@ -287,6 +287,15 @@ dependencies = [
"walkdir",
]
[[package]]
name = "camino"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
dependencies = [
"serde",
]
[[package]]
name = "cast"
version = "0.3.0"
@ -607,6 +616,15 @@ dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.4.1"
@ -642,25 +660,14 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480"
checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys 0.48.0",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "fastrand"
version = "2.0.1"
@ -861,6 +868,29 @@ dependencies = [
"scroll",
]
[[package]]
name = "gourgeist"
version = "0.0.4"
dependencies = [
"camino",
"clap",
"configparser",
"dirs",
"fs-err",
"install-wheel-rs",
"rayon",
"reqwest",
"seahash",
"serde",
"serde_json",
"tempfile",
"thiserror",
"tracing",
"tracing-subscriber",
"wheel-filename",
"which",
]
[[package]]
name = "h2"
version = "0.3.21"
@ -913,6 +943,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "home"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "http"
version = "0.2.9"
@ -1211,9 +1250,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.148"
version = "0.2.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
[[package]]
name = "line-wrap"
@ -1232,9 +1271,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.4.8"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db"
checksum = "45786cec4d5e54a224b15cb9f06751883103a27c19c93eda09b0b4f5f08fefac"
[[package]]
name = "lock_api"
@ -1385,9 +1424,9 @@ dependencies = [
[[package]]
name = "num-traits"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
dependencies = [
"autocfg",
]
@ -1487,7 +1526,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.8",
]
[[package]]
@ -1504,6 +1553,19 @@ dependencies = [
"winapi",
]
[[package]]
name = "parking_lot_core"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.3.5",
"smallvec",
"windows-targets 0.48.5",
]
[[package]]
name = "pep440_rs"
version = "0.3.12"
@ -1793,7 +1855,7 @@ dependencies = [
"indoc 1.0.9",
"libc",
"memoffset",
"parking_lot",
"parking_lot 0.12.1",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
@ -2082,7 +2144,7 @@ dependencies = [
"getrandom",
"http",
"hyper",
"parking_lot",
"parking_lot 0.11.2",
"reqwest",
"reqwest-middleware",
"retry-policies",
@ -2192,6 +2254,12 @@ dependencies = [
"syn 2.0.38",
]
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "security-framework"
version = "2.9.2"
@ -2302,9 +2370,9 @@ dependencies = [
[[package]]
name = "similar"
version = "2.2.1"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf"
checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597"
[[package]]
name = "slab"
@ -2887,7 +2955,7 @@ checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
dependencies = [
"futures",
"js-sys",
"parking_lot",
"parking_lot 0.11.2",
"pin-utils",
"wasm-bindgen",
"wasm-bindgen-futures",
@ -2912,6 +2980,18 @@ dependencies = [
"thiserror",
]
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix",
]
[[package]]
name = "winapi"
version = "0.3.9"

View file

@ -15,12 +15,14 @@ license = "MIT OR Apache-2.0"
anyhow = { version = "1.0.75" }
bitflags = { version = "2.4.0" }
cacache = { version = "11.7.1", default-features = false, features = ["tokio-runtime"] }
camino = { version = "1.1.6", features = ["serde1"] }
clap = { version = "4.4.6" }
colored = { version = "2.0.4" }
configparser = { version = "3.0.2" }
csv = { version = "1.3.0" }
data-encoding = { version = "2.4.0" }
directories = { version = "5.0.1" }
dirs = { version = "5.0.1" }
fs-err = { version = "2.9.0" }
fs2 = { version = "0.4.3" }
futures = { version = "0.3.28" }
@ -39,6 +41,7 @@ reqwest = { version = "0.11.22", features = ["json", "gzip", "brotli", "stream"]
reqwest-middleware = { version = "0.2.3" }
reqwest-retry = { version = "0.3.0" }
rfc2047-decoder = { version = "1.0.1" }
seahash = { version = "4.1.0" }
serde = { version = "1.0.188" }
serde_json = { version = "1.0.107" }
sha2 = { version = "0.10.8" }
@ -53,4 +56,5 @@ tracing-tree = { version = "0.2.5" }
unicode-width = { version = "0.1.8" }
url = { version = "2.4.1" }
walkdir = { version = "2.4.0" }
which = { version = "4.4.2" }
zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

File diff suppressed because it is too large Load diff

View file

@ -8,54 +8,27 @@ license = "MIT OR Apache-2.0"
keywords = ["virtualenv", "venv", "python"]
readme = "Readme.md"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
camino = { version = "1.1.6", features = ["serde1"] }
clap = { version = "4.4.5", features = ["derive"] }
configparser = "3.0.2"
dirs = "5.0.1"
fs-err = "2.9.0"
install-wheel-rs = { version = "0.0.1", optional = true }
minreq = { version = "2.10.0", optional = true, features = ["https"] }
rayon = { version = "1.8.0", optional = true }
seahash = "4.1.0"
serde = { version = "1.0.188", features = ["derive"] }
serde_json = "1.0.107"
tempfile = "3.8.0"
thiserror = "1.0.49"
tracing = "0.1.37"
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
which = "4.4.2"
install-wheel-rs = { path = "../install-wheel-rs", optional = true }
wheel-filename = { path = "../wheel-filename" }
camino = { workspace = true }
clap = { workspace = true }
configparser = { workspace = true }
dirs = { workspace = true }
fs-err = { workspace = true }
reqwest = { workspace = true, optional = true, features = ["blocking"] }
rayon = { workspace = true, optional = true }
seahash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
which = { workspace = true }
[features]
default = ["install"]
install = ["install-wheel-rs", "minreq"]
install = ["install-wheel-rs", "reqwest"]
parallel = ["rayon"]
# zip implementation
[profile.dev.package.adler]
opt-level = 3
[profile.profiling]
inherits = "release"
lto = "thin"
debug = true
# The profile that 'cargo dist' will build with
[profile.dist]
inherits = "release"
lto = "thin"
# Config for 'cargo dist'
[workspace.metadata.dist]
# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.3.1"
# CI backends to support
ci = ["github"]
# The installers to generate for each app
installers = ["shell", "powershell"]
# Target platforms to build apps for (Rust target-triple syntax)
targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"]
# Publish jobs to run in CI
pr-run-mode = "plan"

View file

@ -1,15 +1,17 @@
//! Create a bare virtualenv without any packages install
use crate::interpreter::InterpreterInfo;
use std::io;
use std::io::{BufWriter, Write};
use camino::{Utf8Path, Utf8PathBuf};
use fs_err as fs;
#[cfg(unix)]
use fs_err::os::unix::fs::symlink;
use fs_err::File;
use std::io;
use std::io::{BufWriter, Write};
use tracing::info;
use crate::interpreter::InterpreterInfo;
/// The bash activate scripts with the venv dependent paths patches out
const ACTIVATE_TEMPLATES: &[(&str, &str)] = &[
("activate", include_str!("activator/activate")),
@ -27,27 +29,34 @@ const VIRTUALENV_PATCH: &str = include_str!("_virtualenv.py");
/// Very basic `.cfg` file format writer.
fn write_cfg(f: &mut impl Write, data: &[(&str, String); 8]) -> io::Result<()> {
for (key, value) in data {
writeln!(f, "{} = {}", key, value)?;
writeln!(f, "{key} = {value}")?;
}
Ok(())
}
/// Absolute paths of the virtualenv
#[derive(Debug)]
pub struct VenvPaths {
pub(crate) struct VenvPaths {
/// The location of the virtualenv, e.g. `.venv`
pub root: Utf8PathBuf,
#[allow(unused)]
pub(crate) root: Utf8PathBuf,
/// The python interpreter.rs inside the virtualenv, on unix `.venv/bin/python`
pub interpreter: Utf8PathBuf,
#[allow(unused)]
pub(crate) interpreter: Utf8PathBuf,
/// The directory with the scripts, on unix `.venv/bin`
pub bin: Utf8PathBuf,
#[allow(unused)]
pub(crate) bin: Utf8PathBuf,
/// The site-packages directory where all the packages are installed to, on unix
/// and python 3.11 `.venv/lib/python3.11/site-packages`
pub site_packages: Utf8PathBuf,
#[allow(unused)]
pub(crate) site_packages: Utf8PathBuf,
}
/// Write all the files that belong to a venv without any packages installed.
pub fn create_bare_venv(
pub(crate) fn create_bare_venv(
location: &Utf8Path,
base_python: &Utf8Path,
info: &InterpreterInfo,

View file

@ -1,14 +1,16 @@
use crate::{crate_cache_dir, Error};
use camino::{FromPathBufError, Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
use serde::{Deserialize, Serialize};
use std::io;
use std::io::{BufReader, Write};
use std::process::{Command, Stdio};
use std::time::SystemTime;
use camino::{Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
use serde::{Deserialize, Serialize};
use tracing::{debug, error, info, warn};
use crate::{crate_cache_dir, Error};
const QUERY_PYTHON: &str = include_str!("query_python.py");
#[derive(Clone, Debug, Deserialize, Serialize)]
@ -44,20 +46,19 @@ pub fn get_interpreter_info(interpreter: &Utf8Path) -> Result<InterpreterInfo, E
debug!("Using cache entry {cache_file}");
if modified == cache_entry.modified && interpreter == cache_entry.interpreter {
return Ok(cache_entry.interpreter_info);
} else {
debug!(
"Removing mismatching cache entry {cache_file} ({} {} {} {})",
modified, cache_entry.modified, interpreter, cache_entry.interpreter
);
if let Err(remove_err) = fs::remove_file(&cache_file) {
warn!("Failed to mismatching cache file at {cache_file}: {remove_err}")
}
}
debug!(
"Removing mismatching cache entry {cache_file} ({} {} {} {})",
modified, cache_entry.modified, interpreter, cache_entry.interpreter
);
if let Err(remove_err) = fs::remove_file(&cache_file) {
warn!("Failed to mismatching cache file at {cache_file}: {remove_err}");
}
}
Err(cache_err) => {
debug!("Removing broken cache entry {cache_file} ({cache_err})");
if let Err(remove_err) = fs::remove_file(&cache_file) {
warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})")
warn!("Failed to remove broken cache file at {cache_file}: {remove_err} (original error: {cache_err})");
}
}
}
@ -188,7 +189,7 @@ pub fn parse_python_cli(cli_python: Option<Utf8PathBuf>) -> Result<Utf8PathBuf,
)
})?
.try_into()
.map_err(|err: FromPathBufError| err.into_io_error())?;
.map_err(camino::FromPathBufError::into_io_error)?;
info!("Resolved {python} to {python_in_path}");
python_in_path
};

View file

@ -1,13 +1,15 @@
use crate::bare::create_bare_venv;
use std::io;
use camino::{Utf8Path, Utf8PathBuf};
use dirs::cache_dir;
use interpreter::InterpreterInfo;
use std::io;
use tempfile::PersistError;
use thiserror::Error;
use interpreter::InterpreterInfo;
pub use interpreter::{get_interpreter_info, parse_python_cli};
use crate::bare::create_bare_venv;
mod bare;
mod interpreter;
#[cfg(feature = "install")]
@ -40,7 +42,7 @@ pub enum Error {
},
#[cfg(feature = "install")]
#[error("Failed to contact pypi")]
MinReq(#[from] minreq::Error),
Request(#[from] reqwest::Error),
#[cfg(feature = "install")]
#[error("Failed to install {package}")]
InstallWheel {

View file

@ -1,14 +1,16 @@
use camino::Utf8PathBuf;
use clap::Parser;
use gourgeist::{create_venv, get_interpreter_info, parse_python_cli};
use std::error::Error;
use std::process::ExitCode;
use std::time::Instant;
use camino::Utf8PathBuf;
use clap::Parser;
use tracing::info;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter};
use gourgeist::{create_venv, get_interpreter_info, parse_python_cli};
#[derive(Parser, Debug)]
struct Cli {
path: Option<Utf8PathBuf>,
@ -38,10 +40,17 @@ fn main() -> ExitCode {
let result = run();
info!("Took {}ms", start.elapsed().as_millis());
if let Err(err) = result {
eprintln!("💥 virtualenv creator failed");
#[allow(clippy::print_stderr)]
{
eprintln!("💥 virtualenv creator failed");
}
let mut last_error: Option<&(dyn Error + 'static)> = Some(&err);
while let Some(err) = last_error {
eprintln!(" Caused by: {}", err);
#[allow(clippy::print_stderr)]
{
eprintln!(" Caused by: {err}");
}
last_error = err.source();
}
ExitCode::FAILURE

View file

@ -1,19 +1,23 @@
use crate::bare::VenvPaths;
use crate::interpreter::InterpreterInfo;
use crate::{crate_cache_dir, Error};
use camino::{FromPathBufError, Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
use install_wheel_rs::{install_wheel, InstallLocation, WheelFilename};
#[cfg(feature = "parallel")]
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use std::io;
use std::io::BufWriter;
use std::str::FromStr;
use camino::{Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
#[cfg(feature = "parallel")]
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use tempfile::NamedTempFile;
use tracing::info;
pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
use install_wheel_rs::{install_wheel, InstallLocation};
use wheel_filename::WheelFilename;
use crate::bare::VenvPaths;
use crate::interpreter::InterpreterInfo;
use crate::{crate_cache_dir, Error};
pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
let wheels_cache = crate_cache_dir()?.join("wheels");
let cached_wheel = wheels_cache.join(filename);
if cached_wheel.is_file() {
@ -28,8 +32,8 @@ pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, E
.path()
.to_path_buf()
.try_into()
.map_err(|err: FromPathBufError| err.into_io_error())?;
let mut response = minreq::get(url).send_lazy()?;
.map_err(camino::FromPathBufError::into_io_error)?;
let mut response = reqwest::blocking::get(url)?;
io::copy(&mut response, &mut BufWriter::new(&mut tempfile)).map_err(|err| {
Error::WheelDownload {
url: url.to_string(),
@ -42,7 +46,7 @@ pub fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, E
}
/// Install pip, setuptools and wheel from cache pypi with atm fixed wheels
pub fn install_base_packages(
pub(crate) fn install_base_packages(
location: &Utf8Path,
info: &InterpreterInfo,
paths: &VenvPaths,
@ -71,7 +75,8 @@ pub fn install_base_packages(
install_wheel(
&install_location,
File::open(wheel_file)?,
parsed_filename,
&parsed_filename,
false,
false,
&[],
// Only relevant for monotrail style installation

View file

@ -1,13 +1,15 @@
//! Deprecated, use only as template when implementing caching
use crate::Error;
use std::io;
use std::path::Path;
use camino::{Utf8Path, Utf8PathBuf};
use dirs::data_dir;
use fs_err as fs;
use std::io;
use std::path::Path;
use tracing::debug;
use crate::Error;
/// Install wheel, pip and setuptools from the cache
pub(crate) fn install_base_packages(
bin_dir: &Utf8Path,