diff --git a/Cargo.lock b/Cargo.lock index 14f20c9e0..69adc6480 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -348,7 +348,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d" dependencies = [ - "hashbrown", + "hashbrown 0.12.3", "stacker", ] @@ -638,6 +638,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.5" @@ -655,10 +661,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] -name = "flate2" -version = "1.0.27" +name = "filetime" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.3.5", + "windows-sys 0.48.0", +] + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -883,7 +901,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -899,6 +917,12 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" + [[package]] name = "heck" version = "0.4.1" @@ -1093,7 +1117,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown 0.14.1", + "serde", ] [[package]] @@ -1566,6 +1601,18 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "pep440_rs" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887f66cc62717ea72caac4f1eb4e6f392224da3ffff3f40ec13ab427802746d6" +dependencies = [ + "lazy_static", + "regex", + "serde", + "unicode-width", +] + [[package]] name = "pep508_rs" version = "0.2.3" @@ -1573,7 +1620,7 @@ dependencies = [ "indoc 2.0.4", "log", "once_cell", - "pep440_rs", + "pep440_rs 0.3.12", "pyo3", "pyo3-log", "regex", @@ -1655,7 +1702,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdc0001cfea3db57a2e24bc0d818e9e20e554b5f97fabb9bc231dc240269ae06" dependencies = [ "base64 0.21.4", - "indexmap", + "indexmap 1.9.3", "line-wrap", "quick-xml", "serde", @@ -1730,6 +1777,31 @@ dependencies = [ "thiserror", ] +[[package]] +name = "puffin-build" +version = "0.0.1" +dependencies = [ + "anyhow", + "clap", + "colored", + "flate2", + "fs-err", + "gourgeist", + "indoc 2.0.4", + "pep508_rs", + "pyproject-toml", + "serde", + "serde_json", + "tar", + "tempfile", + "thiserror", + "toml 0.8.2", + "tracing", + "tracing-subscriber", + "which", + "zip", +] + [[package]] name = "puffin-cli" version = "0.0.1" @@ -1745,7 +1817,7 @@ dependencies = [ "indicatif", "install-wheel-rs", "itertools", - "pep440_rs", + "pep440_rs 0.3.12", "pep508_rs", "platform-host", "platform-tags", @@ -1787,7 +1859,7 @@ dependencies = [ "anyhow", "cacache", "install-wheel-rs", - "pep440_rs", + "pep440_rs 0.3.12", "puffin-client", "puffin-interpreter", "puffin-package", @@ -1807,7 +1879,7 @@ version = "0.0.1" dependencies = [ "anyhow", "cacache", - "pep440_rs", + "pep440_rs 0.3.12", "pep508_rs", "platform-host", "puffin-package", @@ -1827,7 +1899,7 @@ dependencies = [ "mailparse", "memchr", "once_cell", - "pep440_rs", + "pep440_rs 0.3.12", "pep508_rs", "regex", "rfc2047-decoder", @@ -1848,7 +1920,7 @@ dependencies = [ "bitflags 2.4.0", "futures", "once_cell", - "pep440_rs", + "pep440_rs 0.3.12", "pep508_rs", "platform-host", "platform-tags", @@ -1933,6 +2005,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "pyproject-toml" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "569e259cd132eb8cec5df8b672d187c5260f82ad352156b5da9549d4472e64b0" +dependencies = [ + "indexmap 2.0.2", + "pep440_rs 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "pep508_rs", + "serde", + "toml 0.7.8", +] + [[package]] name = "quick-xml" version = "0.29.0" @@ -2337,6 +2422,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2511,6 +2605,17 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.11" @@ -2729,6 +2834,65 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + +[[package]] +name = "toml" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.20.2", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.0.2", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap 2.0.2", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -3256,6 +3420,15 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winnow" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -3266,6 +3439,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "xattr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +dependencies = [ + "libc", +] + [[package]] name = "xxhash-rust" version = "0.8.7" diff --git a/Cargo.toml b/Cargo.toml index b45be2549..16e28d4de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ csv = { version = "1.3.0" } data-encoding = { version = "2.4.0" } directories = { version = "5.0.1" } dirs = { version = "5.0.1" } +flate2 = { version = "1.0.28" } fs-err = { version = "2.9.0" } fs2 = { version = "0.4.3" } futures = { version = "0.3.28" } @@ -31,12 +32,14 @@ glibc_version = { version = "0.1.2" } goblin = { version = "0.7.1" } http-cache-reqwest = { version = "0.11.3" } indicatif = { version = "0.17.7" } +indoc = { version = "2.0.4" } itertools = { version = "0.11.0" } mailparse = { version = "0.14.0" } memchr = { version = "2.6.4" } once_cell = { version = "1.18.0" } platform-info = { version = "2.0.2" } plist = { version = "1.5.0" } +pyproject-toml = { version = "0.7.0" } rayon = { version = "1.8.0" } reflink-copy = { version = "0.1.9" } regex = { version = "1.9.6" } @@ -48,11 +51,13 @@ seahash = { version = "4.1.0" } serde = { version = "1.0.188" } serde_json = { version = "1.0.107" } sha2 = { version = "0.10.8" } +tar = { version = "0.4.40" } target-lexicon = { version = "0.12.11" } tempfile = { version = "3.8.0" } thiserror = { version = "1.0.49" } tokio = { version = "1.16.1", features = ["rt-multi-thread"] } tokio-util = { version = "0.7.9", features = ["compat"] } +toml = { version = "0.8.2" } tracing = { version = "0.1.37" } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } tracing-tree = { version = "0.2.5" } @@ -64,6 +69,10 @@ walkdir = { version = "2.4.0" } which = { version = "4.4.2" } zip = { version = "0.6.6", default-features = false, features = ["deflate"] } +[patch.crates-io] +# For pyproject-toml +pep508_rs = { path = "crates/pep508-rs" } + [profile.profiling] inherits = "release" debug = true diff --git a/crates/gourgeist/src/interpreter.rs b/crates/gourgeist/src/interpreter.rs index 8b9702199..bdddb2ac6 100644 --- a/crates/gourgeist/src/interpreter.rs +++ b/crates/gourgeist/src/interpreter.rs @@ -1,5 +1,6 @@ use std::io; use std::io::{BufReader, Write}; +use std::path::Path; use std::process::{Command, Stdio}; use std::time::SystemTime; @@ -7,7 +8,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use fs_err as fs; use fs_err::File; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; use crate::{crate_cache_dir, Error}; @@ -23,9 +24,7 @@ pub struct InterpreterInfo { } /// Gets the interpreter.rs info, either cached or by running it. -pub fn get_interpreter_info( - interpreter: impl AsRef, -) -> Result { +pub fn get_interpreter_info(interpreter: impl AsRef) -> Result { let interpreter = Utf8Path::from_path(interpreter.as_ref()) .ok_or_else(|| Error::NonUTF8Path(interpreter.as_ref().to_path_buf()))?; @@ -171,7 +170,7 @@ pub fn parse_python_cli(cli_python: Option) -> Result) -> Result 1 { // Does this path contain a slash (unix) or backslash (windows)? In that case, assume it's // relative or absolute path that we don't need to resolve - info!("Assuming {python} is a path"); + debug!("Assuming {python} is a path"); python } else { let python_in_path = which::which(python.as_std_path()) @@ -195,7 +194,7 @@ pub fn parse_python_cli(cli_python: Option) -> Result &Self::Target { + &self.0 + } +} + +impl Venv { + pub fn new(location: impl Into) -> Result { + let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?; + Ok(Self(location)) + } + + /// Returns the location of the python interpreter + pub fn python_interpreter(&self) -> PathBuf { + #[cfg(unix)] + { + self.0.join("bin").join("python").into_std_path_buf() + } + #[cfg(windows)] + { + self.0 + .join("Scripts") + .join("python.exe") + .into_std_path_buf() + } + #[cfg(not(any(unix, windows)))] + { + compile_error!("Only windows and unix (linux, mac os, etc.) are supported") + } + } } pub(crate) fn crate_cache_dir() -> io::Result { @@ -63,22 +101,21 @@ pub(crate) fn crate_cache_dir() -> io::Result { /// Create a virtualenv and if not bare, install `wheel`, `pip` and `setuptools`. pub fn create_venv( - location: impl AsRef, - base_python: impl AsRef, + location: impl Into, + base_python: impl AsRef, info: &InterpreterInfo, bare: bool, -) -> Result<(), Error> { - let location = Utf8Path::from_path(location.as_ref()) - .ok_or_else(|| Error::NonUTF8Path(location.as_ref().to_path_buf()))?; +) -> Result { + let location = Utf8PathBuf::from_path_buf(location.into()).map_err(Error::NonUTF8Path)?; let base_python = Utf8Path::from_path(base_python.as_ref()) .ok_or_else(|| Error::NonUTF8Path(base_python.as_ref().to_path_buf()))?; - let paths = create_bare_venv(location, base_python, info)?; + let paths = create_bare_venv(&location, base_python, info)?; if !bare { #[cfg(feature = "install")] { - packages::install_base_packages(location, info, &paths)?; + packages::install_base_packages(&location, info, &paths)?; } #[cfg(not(feature = "install"))] { @@ -90,5 +127,5 @@ pub fn create_venv( } } - Ok(()) + Ok(Venv(location)) } diff --git a/crates/gourgeist/src/packages.rs b/crates/gourgeist/src/packages.rs index 1410a69dc..4a1ef2879 100644 --- a/crates/gourgeist/src/packages.rs +++ b/crates/gourgeist/src/packages.rs @@ -8,7 +8,7 @@ use fs_err::File; #[cfg(feature = "parallel")] use rayon::iter::{IntoParallelIterator, ParallelIterator}; use tempfile::NamedTempFile; -use tracing::info; +use tracing::debug; use install_wheel_rs::{install_wheel, InstallLocation}; use wheel_filename::WheelFilename; @@ -21,11 +21,11 @@ pub(crate) fn download_wheel_cached(filename: &str, url: &str) -> Result + +use anyhow::Context; +use flate2::read::GzDecoder; +use fs_err as fs; +use fs_err::{DirEntry, File}; +use gourgeist::{InterpreterInfo, Venv}; +use indoc::formatdoc; +use pep508_rs::Requirement; +use pyproject_toml::PyProjectToml; +use std::io; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use tar::Archive; +use tempfile::tempdir; +use thiserror::Error; +use tracing::{debug, instrument}; +use zip::ZipArchive; + +#[derive(Error, Debug)] +pub enum Error { + #[error(transparent)] + IO(#[from] io::Error), + #[error("Failed to read zip file")] + Zip(#[from] zip::result::ZipError), + #[error("Unsupported archive format (extension not recognized): {0}")] + UnsupportedArchiveType(String), + #[error("Invalid source distribution: {0}")] + InvalidSourceDistribution(String), + #[error("Invalid pyproject.toml")] + PyprojectTomlInvalid(#[from] toml::de::Error), + #[error("Failed to install requirements")] + RequirementsInstall(#[source] anyhow::Error), + #[error("Failed to create temporary virtual environment")] + Gourgeist(#[from] gourgeist::Error), + #[error("Failed to run {0}")] + CommandFailed(PathBuf, #[source] io::Error), + #[error("{message}:\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---")] + BuildBackend { + message: String, + stdout: String, + stderr: String, + }, +} + +impl Error { + fn from_command_output(message: String, output: &Output) -> Self { + Self::BuildBackend { + message, + stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), + } + } +} + +#[instrument(skip_all)] +fn resolve_and_install(venv: impl AsRef, requirements: &[Requirement]) -> anyhow::Result<()> { + debug!("Calling pip to install build dependencies"); + let python = Venv::new(venv.as_ref())?.python_interpreter(); + // No error handling because we want have to replace this with the real resolver and installer + // anyway. + let installation = Command::new(python) + .args(["-m", "pip", "install"]) + .args( + requirements + .iter() + .map(ToString::to_string) + .collect::>(), + ) + .output() + .context("pip install failed")?; + if !installation.status.success() { + anyhow::bail!("Installation failed :(") + } + Ok(()) +} + +/// Returns the directory with the `pyproject.toml`/`setup.py` +#[instrument(skip_all, fields(path))] +fn extract_archive(path: &Path, extracted: &PathBuf) -> Result { + // TODO(konstin): Simplify this with camino paths? + if path.extension().is_some_and(|extension| extension == "zip") { + let mut archive = ZipArchive::new(File::open(path)?)?; + archive.extract(extracted)?; + // .tar.gz + } else if path.extension().is_some_and(|extension| extension == "gz") + && path.file_stem().is_some_and(|stem| { + Path::new(stem) + .extension() + .is_some_and(|extension| extension == "tar") + }) + { + let mut archive = Archive::new(GzDecoder::new(File::open(path)?)); + archive.unpack(extracted)?; + } else { + return Err(Error::UnsupportedArchiveType( + path.file_name() + .unwrap_or(path.as_os_str()) + .to_string_lossy() + .to_string(), + )); + } + + // > A .tar.gz source distribution (sdist) contains a single top-level directory called + // > `{name}-{version}` (e.g. foo-1.0), containing the source files of the package. + // TODO(konstin): Verify the name of the directory + let top_level = fs::read_dir(extracted)?.collect::>>()?; + let [root] = top_level.as_slice() else { + return Err(Error::InvalidSourceDistribution(format!( + "The top level of the archive must only contain a list directory, but it contains {top_level:?}" + ))); + }; + Ok(root.path()) +} + +#[instrument(skip(script, root))] +fn run_python_script( + python_interpreter: &PathBuf, + script: &String, + root: &Path, +) -> Result { + Command::new(python_interpreter) + .args(["-c", script]) + .current_dir(root) + .output() + .map_err(|err| Error::CommandFailed(python_interpreter.clone(), err)) +} + +/// Returns `Ok(None)` if this is not a pyproject.toml build +fn pep517_build( + wheel_dir: &Path, + root: &Path, + temp_dir: &Path, + base_python: &Path, + data: &InterpreterInfo, +) -> Result, Error> { + if !root.join("pyproject.toml").is_file() { + // We'll try setup.py instead + return Ok(None); + } + // TODO(konstin): Create bare venvs when we don't need pip anymore + let venv = gourgeist::create_venv(temp_dir.join("venv"), base_python, data, false)?; + let pyproject_toml: PyProjectToml = + toml::from_str(&fs::read_to_string(root.join("pyproject.toml"))?) + .map_err(Error::PyprojectTomlInvalid)?; + let mut requirements = pyproject_toml.build_system.requires; + resolve_and_install(venv.deref().as_std_path(), &requirements) + .map_err(Error::RequirementsInstall)?; + let Some(backend) = &pyproject_toml.build_system.build_backend else { + // > If the pyproject.toml file is absent, or the build-backend key is missing, the + // > source tree is not using this specification, and tools should revert to the legacy + // > behaviour of running setup.py (either directly, or by implicitly invoking the + // > setuptools.build_meta:__legacy__ backend). + return Ok(None); + }; + let backend_import = if let Some((path, object)) = backend.split_once(':') { + format!("from {path} import {object}") + } else { + format!("import {backend}") + }; + + debug!("Calling `{}.get_requires_for_build_wheel()`", backend); + let script = formatdoc! { + r#"{} as backend + import json + + if get_requires_for_build_wheel := getattr(backend, "get_requires_for_build_wheel", None): + requires = get_requires_for_build_wheel() + else: + requires = [] + print(json.dumps(requires)) + "#, backend_import + }; + let python_interpreter = venv.python_interpreter(); + let output = run_python_script(&python_interpreter, &script, root)?; + if !output.status.success() { + return Err(Error::from_command_output( + "Build backend failed to determine extras requires with `get_requires_for_build_wheel`" + .to_string(), + &output, + )); + } + let stdout = String::from_utf8_lossy(&output.stdout); + + let extra_requires: Vec = + serde_json::from_str(stdout.lines().last().unwrap_or_default()).map_err(|err| { + Error::from_command_output( + format!( + "Build backend failed to return extras requires with \ + `get_requires_for_build_wheel`: {err}" + ), + &output, + ) + })?; + // Some packages (such as tqdm 4.66.1) list only extra requires that have already been part of + // the pyproject.toml requires (in this case, `wheel`). We can skip doing the whole resolution + // and installation again. + // TODO(konstin): Do we still need this when we have a fast resolver? + if !extra_requires.is_empty() && !extra_requires.iter().all(|req| requirements.contains(req)) { + debug!("Installing extra requirements for build backend"); + // TODO(konstin): Do we need to resolve them together? + requirements.extend(extra_requires); + resolve_and_install(&*venv, &requirements).map_err(Error::RequirementsInstall)?; + } + + debug!("Calling `{}.build_wheel()`", backend); + let escaped_wheel_dir = wheel_dir + .display() + .to_string() + .replace('\\', "\\\\") + .replace('"', "\\\""); + let script = formatdoc! { + r#"{} as backend + print(backend.build_wheel("{}")) + "#, backend_import, escaped_wheel_dir + }; + let output = run_python_script(&python_interpreter, &script, root)?; + if !output.status.success() { + return Err(Error::from_command_output( + "Build backend failed to build wheel through `build_wheel()` ".to_string(), + &output, + )); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let wheel = stdout + .lines() + .last() + .map(|wheel_filename| wheel_dir.join(wheel_filename)); + let Some(wheel) = wheel.filter(|wheel| wheel.is_file()) else { + return Err(Error::from_command_output( + "Build backend did not return the wheel filename through `build_wheel()`".to_string(), + &output, + )); + }; + Ok(Some(wheel)) +} + +/// Build a source distribution from an archive (`.zip` or `.tar.gz`), return the location of the +/// built wheel. +/// +/// The location will be inside `temp_dir`, i.e. you must use the wheel before dropping the temp +/// dir. +/// +/// +#[instrument(skip(wheel_dir, interpreter_info))] +pub fn build_sdist( + path: &Path, + wheel_dir: &Path, + base_python: &Path, + interpreter_info: &InterpreterInfo, +) -> Result { + debug!("Building {}", path.display()); + // TODO(konstin): Parse and verify filenames + let temp_dir = tempdir()?; + let temp_dir = temp_dir.path(); + // The build scripts run with the extracted root as cwd, so they need the absolute path + let wheel_dir = fs::canonicalize(wheel_dir)?; + + let extracted = temp_dir.join("extracted"); + let root = extract_archive(path, &extracted)?; + + let wheel = pep517_build(&wheel_dir, &root, temp_dir, base_python, interpreter_info)?; + + if let Some(wheel) = wheel { + Ok(wheel) + } else if root.join("setup.py").is_file() { + let venv = + gourgeist::create_venv(temp_dir.join("venv"), base_python, interpreter_info, false)?; + let python_interpreter = venv.python_interpreter(); + let output = Command::new(&python_interpreter) + .args(["setup.py", "bdist_wheel"]) + .current_dir(&root) + .output() + .map_err(|err| Error::CommandFailed(python_interpreter.clone(), err))?; + if !output.status.success() { + return Err(Error::from_command_output( + "Failed building wheel through setup.py".to_string(), + &output, + )); + } + let dist = fs::read_dir(root.join("dist"))?; + let dist_dir = dist.collect::>>()?; + let [dist_wheel] = dist_dir.as_slice() else { + return Err(Error::from_command_output( + format!( + "Expected exactly wheel in `dist/` after invoking setup.py, found {dist_dir:?}" + ), + &output, + )); + }; + // TODO(konstin): Faster copy such as reflink? Or maybe don't really let the user pick the target dir + let wheel = wheel_dir.join(dist_wheel.file_name()); + fs::copy(dist_wheel.path(), &wheel)?; + // TODO(konstin): Check wheel filename + Ok(wheel) + } else { + Err(Error::InvalidSourceDistribution( + "The archive contains neither a pyproject.toml or a setup.py at the top level" + .to_string(), + )) + } +} diff --git a/crates/puffin-build/src/main.rs b/crates/puffin-build/src/main.rs new file mode 100644 index 000000000..9d8821622 --- /dev/null +++ b/crates/puffin-build/src/main.rs @@ -0,0 +1,71 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] + +use anyhow::Context; +use clap::Parser; +use colored::Colorize; +use fs_err as fs; +use puffin_build::{build_sdist, Error}; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Instant; +use std::{env, io}; +use tracing::debug; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{fmt, EnvFilter}; + +#[derive(Parser)] +struct Args { + /// Base python in a way that can be found with `which` + /// TODO(konstin): Also use proper python parsing here + #[clap(short, long)] + python: Option, + /// Directory to story the built wheel in + #[clap(short, long)] + wheels: Option, + sdist: PathBuf, +} + +fn run() -> anyhow::Result<()> { + let args = Args::parse(); + let wheel_dir = if let Some(wheel_dir) = args.wheels { + fs::create_dir_all(&wheel_dir).context("Invalid wheel directory")?; + wheel_dir + } else { + env::current_dir()? + }; + + // TODO: That's no way to deal with paths in PATH + let base_python = which::which(args.python.unwrap_or("python3".into())).map_err(|err| { + Error::IO(io::Error::new( + io::ErrorKind::NotFound, + format!("Can't find `python3` ({err})"), + )) + })?; + let interpreter_info = gourgeist::get_interpreter_info(&base_python)?; + + let wheel = build_sdist(&args.sdist, &wheel_dir, &base_python, &interpreter_info)?; + println!("Wheel built to {}", wheel.display()); + Ok(()) +} + +fn main() -> ExitCode { + tracing_subscriber::registry() + .with(fmt::layer().with_span_events(FmtSpan::CLOSE)) + .with(EnvFilter::from_default_env()) + .init(); + + let start = Instant::now(); + let result = run(); + debug!("Took {}ms", start.elapsed().as_millis()); + if let Err(err) = result { + eprintln!("{}", "puffin-build failed".red().bold()); + for err in err.chain() { + eprintln!(" {}: {}", "Caused by".red().bold(), err); + } + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} diff --git a/crates/puffin-build/test.sh b/crates/puffin-build/test.sh new file mode 100644 index 000000000..252156225 --- /dev/null +++ b/crates/puffin-build/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +mkdir -p downloads +if [ ! -f downloads/tqdm-4.66.1.tar.gz ]; then + wget https://files.pythonhosted.org/packages/62/06/d5604a70d160f6a6ca5fd2ba25597c24abd5c5ca5f437263d177ac242308/tqdm-4.66.1.tar.gz -O downloads/tqdm-4.66.1.tar.gz +fi +if [ ! -f downloads/geoextract-0.3.1.tar.gz ]; then + wget https://files.pythonhosted.org/packages/c4/00/9d9826a6e1c9139cc7183647f47f6b7acb290fa4c572140aa84a12728e60/geoextract-0.3.1.tar.gz -O downloads/geoextract-0.3.1.tar.gz +fi +RUST_LOG=puffin_build=debug cargo run -p puffin-build --bin puffin-build -- --wheels wheels downloads/tqdm-4.66.1.tar.gz +RUST_LOG=puffin_build=debug cargo run -p puffin-build --bin puffin-build -- --wheels wheels downloads/geoextract-0.3.1.tar.gz + +# Check that pip accepts the wheels. It would be better to do functional checks +virtualenv -p 3.8 -q --clear wheels/.venv +wheels/.venv/bin/pip install -q --no-deps wheels/geoextract-0.3.1-py3-none-any.whl +wheels/.venv/bin/pip install -q --no-deps wheels/tqdm-4.66.1-py3-none-any.whl \ No newline at end of file