Remove virtualenv setup from gourgeist (#339)

We now only support building bare environments.
This commit is contained in:
Charlie Marsh 2023-11-06 10:32:45 -08:00 committed by GitHub
parent b013ea9c93
commit a5e535f6fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 4 additions and 298 deletions

18
Cargo.lock generated
View file

@ -753,15 +753,6 @@ dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.4.1"
@ -1094,18 +1085,11 @@ version = "0.0.4"
dependencies = [
"camino",
"clap",
"configparser",
"dirs",
"distribution-filename",
"fs-err",
"install-wheel-rs",
"platform-host",
"puffin-interpreter",
"rayon",
"reqwest",
"serde",
"serde_json",
"tempfile",
"thiserror",
"tracing",
"tracing-subscriber",
@ -2168,14 +2152,12 @@ name = "puffin-build"
version = "0.0.1"
dependencies = [
"anyhow",
"clap",
"flate2",
"fs-err",
"gourgeist",
"indoc",
"pep508_rs",
"platform-host",
"platform-tags",
"puffin-interpreter",
"puffin-traits",
"pyproject-toml",

View file

@ -14,27 +14,15 @@ authors = { workspace = true }
license = { workspace = true }
[dependencies]
distribution-filename = { path = "../distribution-filename" }
install-wheel-rs = { path = "../install-wheel-rs", optional = true }
platform-host = { path = "../platform-host" }
puffin-interpreter = { path = "../puffin-interpreter" }
camino = { workspace = true }
clap = { workspace = true }
configparser = { workspace = true }
dirs = { workspace = true }
fs-err = { workspace = true }
reqwest = { workspace = true, optional = true, default-features = false, features = ["blocking"] }
rayon = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
which = { workspace = true }
[features]
default = ["install"]
install = ["install-wheel-rs", "reqwest"]
parallel = ["rayon"]

View file

@ -2,8 +2,6 @@ use std::io;
use std::path::{Path, PathBuf};
use camino::{Utf8Path, Utf8PathBuf};
use dirs::cache_dir;
use tempfile::PersistError;
use thiserror::Error;
pub use interpreter::parse_python_cli;
@ -14,84 +12,28 @@ use crate::bare::create_bare_venv;
mod bare;
mod interpreter;
#[cfg(feature = "install")]
mod packages;
#[cfg(not(feature = "install"))]
mod virtualenv_cache;
#[derive(Debug, Error)]
pub enum Error {
#[error(transparent)]
IO(#[from] io::Error),
/// It's effectively an io error with extra info
#[error(transparent)]
Persist(#[from] PersistError),
/// Adds url and target path to the io error
#[error("Failed to download wheel from {url} to {path}")]
WheelDownload {
url: String,
path: Utf8PathBuf,
#[source]
err: io::Error,
},
#[error("Failed to determine python interpreter to use")]
InvalidPythonInterpreter(#[source] Box<dyn std::error::Error + Sync + Send>),
#[error("Failed to query python interpreter at {interpreter}")]
PythonSubcommand {
interpreter: Utf8PathBuf,
#[source]
err: io::Error,
},
#[cfg(feature = "install")]
#[error("Failed to contact pypi")]
Request(#[from] reqwest::Error),
#[cfg(feature = "install")]
#[error("Failed to install {package}")]
InstallWheel {
package: String,
#[source]
err: install_wheel_rs::Error,
},
#[error("{0} is not a valid UTF-8 path")]
NonUTF8Path(PathBuf),
#[error(transparent)]
Platform(#[from] PlatformError),
}
pub(crate) fn crate_cache_dir() -> io::Result<Utf8PathBuf> {
Ok(cache_dir()
.and_then(|path| Utf8PathBuf::from_path_buf(path).ok())
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Couldn't detect cache dir"))?
.join(env!("CARGO_PKG_NAME")))
}
/// Create a virtualenv and if not bare, install `wheel`, `pip` and `setuptools`.
/// Create a virtualenv.
pub fn create_venv(
location: impl Into<PathBuf>,
base_python: impl AsRef<Path>,
info: &InterpreterInfo,
bare: bool,
) -> Result<Virtualenv, Error> {
let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?;
let base_python = Utf8Path::from_path(base_python.as_ref())
.ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?;
let paths = create_bare_venv(&location, base_python, info)?;
if !bare {
#[cfg(feature = "install")]
{
packages::install_base_packages(&location, info, &paths)?;
}
#[cfg(not(feature = "install"))]
{
virtualenv_cache::install_base_packages(
&paths.bin,
&paths.interpreter,
&paths.site_packages,
)?;
}
}
Ok(Virtualenv::new_prefix(location.as_std_path(), info))
Ok(Virtualenv::new_prefix(paths.root.as_std_path(), info))
}

View file

@ -17,8 +17,6 @@ struct Cli {
path: Option<Utf8PathBuf>,
#[clap(short, long)]
python: Option<Utf8PathBuf>,
#[clap(long)]
bare: bool,
}
fn run() -> Result<(), gourgeist::Error> {
@ -27,7 +25,7 @@ fn run() -> Result<(), gourgeist::Error> {
let python = parse_python_cli(cli.python)?;
let platform = Platform::current()?;
let info = InterpreterInfo::query(python.as_std_path(), platform, None).unwrap();
create_venv(location, &python, &info, cli.bare)?;
create_venv(location, &python, &info)?;
Ok(())
}

View file

@ -1,90 +0,0 @@
use std::io;
use std::io::BufWriter;
use std::str::FromStr;
use camino::{Utf8Path, Utf8PathBuf};
use fs_err as fs;
use fs_err::File;
#[cfg(feature = "parallel")]
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use tempfile::NamedTempFile;
use tracing::debug;
use distribution_filename::WheelFilename;
use install_wheel_rs::{install_wheel, InstallLocation};
use puffin_interpreter::InterpreterInfo;
use crate::bare::VenvPaths;
use crate::{crate_cache_dir, Error};
pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result<Utf8PathBuf, Error> {
let wheels_cache = crate_cache_dir()?.join("wheels");
let cached_wheel = wheels_cache.join(filename);
if cached_wheel.is_file() {
debug!("Using cached wheel at {cached_wheel}");
return Ok(cached_wheel);
}
debug!("Downloading wheel from {url} to {cached_wheel}");
fs::create_dir_all(&wheels_cache)?;
let mut tempfile = NamedTempFile::new_in(wheels_cache)?;
let tempfile_path: Utf8PathBuf = tempfile
.path()
.to_path_buf()
.try_into()
.map_err(camino::FromPathBufError::into_io_error)?;
let mut response = reqwest::blocking::get(url)?;
io::copy(&mut response, &mut BufWriter::new(&mut tempfile)).map_err(|err| {
Error::WheelDownload {
url: url.to_string(),
path: tempfile_path.to_path_buf(),
err,
}
})?;
tempfile.persist(&cached_wheel)?;
Ok(cached_wheel)
}
/// Install pip, setuptools and wheel from cache pypi with atm fixed wheels
pub(crate) fn install_base_packages(
location: &Utf8Path,
info: &InterpreterInfo,
paths: &VenvPaths,
) -> Result<(), Error> {
let install_location = InstallLocation::new(location.canonicalize()?, info.simple_version());
let install_location = install_location.acquire_lock()?;
// TODO(konstin): Use the json api instead
// TODO(konstin): Only check the json API so often (monthly? daily?)
let packages = [
("pip-23.2.1-py3-none-any.whl", "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl"),
("setuptools-68.2.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/bb/26/7945080113158354380a12ce26873dd6c1ebd88d47f5bc24e2c5bb38c16a/setuptools-68.2.2-py3-none-any.whl"),
("wheel-0.41.2-py3-none-any.whl", "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl"),
];
#[cfg(feature = "rayon")]
let iterator = packages.into_par_iter();
#[cfg(not(feature = "rayon"))]
let iterator = packages.into_iter();
iterator
.map(|(filename, url)| {
let wheel_file = download_wheel_cached(filename, url)?;
let parsed_filename = WheelFilename::from_str(filename).unwrap();
install_wheel(
&install_location,
File::open(wheel_file)?,
&parsed_filename,
None,
false,
false,
&[],
paths.interpreter.as_std_path(),
)
.map_err(|err| Error::InstallWheel {
package: filename.to_string(),
err,
})?;
Ok(())
})
.collect::<Result<Vec<()>, Error>>()?;
Ok(())
}

View file

@ -1,111 +0,0 @@
//! Deprecated, use only as template when implementing caching
use std::io;
use std::path::Path;
use camino::{Utf8Path, Utf8PathBuf};
use dirs::data_dir;
use fs_err as fs;
use tracing::debug;
use crate::Error;
/// Install wheel, pip and setuptools from the cache
pub(crate) fn install_base_packages(
bin_dir: &Utf8Path,
venv_python: &Utf8Path,
site_packages: &Utf8Path,
) -> Result<(), Error> {
// Install packages
// TODO: Implement our own logic:
// * Our own cache and logic to detect whether a wheel is present
// * Check if the version is recent (e.g. update if older than 1 month)
// * Query pypi API if no, parse versions (pep440) and their metadata
// * Download compatible wheel (py3-none-any should do)
// * Install into the cache directory
let prefix = "virtualenv/wheel/3.11/image/1/CopyPipInstall/";
let wheel_tag = "py3-none-any";
let packages = &[
("pip", "23.2.1"),
("setuptools", "68.2.2"),
("wheel", "0.41.2"),
];
let virtualenv_data_dir: Utf8PathBuf = data_dir().unwrap().try_into().unwrap();
for (name, version) in packages {
// TODO: acquire lock
let unpacked_wheel = virtualenv_data_dir
.join(prefix)
.join(format!("{name}-{version}-{wheel_tag}"));
debug!("Installing {name} by copying from {unpacked_wheel}");
copy_dir_all(&unpacked_wheel, site_packages.as_std_path())?;
// Generate launcher
// virtualenv for some reason creates extra entrypoints that we don't
// https://github.com/pypa/virtualenv/blob/025e96fbad37f85617364002ae2a0064b09fc984/src/virtualenv/seed/embed/via_app_data/pip_install/base.py#L74-L95
let ini_text = fs::read_to_string(
site_packages
.join(format!("{name}-{version}.dist-info"))
.join("entry_points.txt"),
)?;
let entry_points_mapping = configparser::ini::Ini::new_cs()
.read(ini_text)
.map_err(|err| format!("{name} entry_points.txt is invalid: {}", err))
.unwrap();
for (key, value) in entry_points_mapping
.get("console_scripts")
.cloned()
.unwrap_or_default()
{
let (import_from, function) = value
.as_ref()
.and_then(|value| value.split_once(':'))
.ok_or_else(|| {
format!("{name} entry_points.txt {key} has an invalid value {value:?}")
})
.unwrap();
let launcher = bin_dir.join(key);
let launcher_script = unix_launcher_script(venv_python, import_from, function);
fs::write(&launcher, launcher_script)?;
// We need to make the launcher executable
#[cfg(target_family = "unix")]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(launcher, std::fs::Permissions::from_mode(0o755))?;
}
}
}
Ok(())
}
/// https://stackoverflow.com/a/65192210/3549270
pub fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(&dst)?;
for entry in fs::read_dir(src.as_ref())? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
}
}
Ok(())
}
/// Template for the console scripts in the `bin` directory
pub fn unix_launcher_script(python: &Utf8Path, import_from: &str, function: &str) -> String {
format!(
r#"#!{python}
# -*- coding: utf-8 -*-
import re
import sys
from {import_from} import {function}
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit({function}())
"#,
python = python,
import_from = import_from,
function = function
)
}

View file

@ -14,12 +14,10 @@ license = { workspace = true }
gourgeist = { path = "../gourgeist" }
pep508_rs = { path = "../pep508-rs" }
platform-host = { path = "../platform-host" }
platform-tags = { path = "../platform-tags" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-traits = { path = "../puffin-traits" }
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"] }
flate2 = { workspace = true }
fs-err = { workspace = true }
indoc = { workspace = true }

View file

@ -166,7 +166,6 @@ impl SourceDistributionBuild {
temp_dir.path().join(".venv"),
build_context.base_python(),
interpreter_info,
true,
)?;
// There are packages such as DTLSSocket 0.1.16 that say

View file

@ -87,7 +87,7 @@ fn venv_impl(
.into_diagnostic()?;
// Create the virtual environment.
gourgeist::create_venv(path, &base_python, &interpreter_info, true)
gourgeist::create_venv(path, &base_python, &interpreter_info)
.map_err(VenvError::CreationError)?;
Ok(ExitStatus::Success)