From 8cc4fe0d44e444c29125ddf9a3f96c420f24dbb4 Mon Sep 17 00:00:00 2001 From: konsti Date: Wed, 18 Oct 2023 21:11:17 +0200 Subject: [PATCH] Install source distribution requirements with puffin itself instead of pip (#122) This is also a lot faster. Unfortunately it copies a lot of code from the sync cli since the `Printer` is private. The first commit are some refactorings i made when i thought about how i could reuse the existing code. --- Cargo.lock | 138 +++++++++--------- crates/puffin-build/Cargo.toml | 15 +- crates/puffin-build/src/lib.rs | 119 ++++++++++++---- crates/puffin-build/src/main.rs | 15 +- crates/puffin-cli/src/commands/compile.rs | 8 +- crates/puffin-cli/src/commands/sync.rs | 166 +++++++++++++--------- crates/puffin-client/src/client.rs | 9 +- crates/puffin-interpreter/Cargo.toml | 2 +- crates/puffin-interpreter/src/lib.rs | 13 ++ scripts/benchmarks/sync.sh | 14 +- 10 files changed, 306 insertions(+), 193 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 583bb42fa..4f2a23242 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,9 +138,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-compression" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" dependencies = [ "brotli", "flate2", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", @@ -220,9 +220,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "block-buffer" @@ -571,12 +571,9 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" -dependencies = [ - "powerfmt", -] +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "digest" @@ -1083,16 +1080,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows 0.48.0", ] [[package]] @@ -1309,9 +1306,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1509,7 +1506,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1583,7 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.8", ] [[package]] @@ -1602,13 +1599,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.3.5", "smallvec", "windows-targets 0.48.5", ] @@ -1741,12 +1738,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1815,18 +1806,29 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", + "directories", "flate2", "fs-err", "gourgeist", "indoc 2.0.4", + "itertools", "owo-colors", "pep508_rs", + "platform-host", + "platform-tags", + "puffin-client", + "puffin-installer", + "puffin-interpreter", + "puffin-package", + "puffin-resolver", + "puffin-workspace", "pyproject-toml", "serde", "serde_json", "tar", "tempfile", "thiserror", + "tokio", "toml 0.8.2", "tracing", "tracing-subscriber", @@ -1839,7 +1841,7 @@ name = "puffin-cli" version = "0.0.1" dependencies = [ "anyhow", - "bitflags 2.4.1", + "bitflags 2.4.0", "cacache", "clap", "directories", @@ -1956,7 +1958,7 @@ name = "puffin-resolver" version = "0.0.1" dependencies = [ "anyhow", - "bitflags 2.4.1", + "bitflags 2.4.0", "futures", "once_cell", "pep440_rs 0.3.12", @@ -2026,9 +2028,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c09c2b349b6538d8a73d436ca606dab6ce0aaab4dad9e6b7bdd57a4f556c3bc3" +checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605" dependencies = [ "arc-swap", "log", @@ -2163,15 +2165,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_users" version = "0.4.3" @@ -2185,25 +2178,25 @@ dependencies = [ [[package]] name = "reflink-copy" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9bd37fcf997c2d9ec7ebdff893c396677664164cf72105b063ac4a483702d3" +checksum = "d7e3e017e993f86feeddf8a7fb609ca49f89082309e328e27aefd4a25bb317a4" dependencies = [ "cfg-if 1.0.0", "ioctl-sys", - "windows", + "windows 0.51.1", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "regex-automata 0.4.1", + "regex-syntax 0.8.1", ] [[package]] @@ -2217,13 +2210,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.1", ] [[package]] @@ -2234,9 +2227,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" [[package]] name = "reqwest" @@ -2357,11 +2350,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", @@ -2455,18 +2448,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.189" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -2837,13 +2830,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", - "powerfmt", "serde", "time-core", "time-macros", @@ -3019,10 +3011,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -3031,9 +3024,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", @@ -3042,9 +3035,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -3383,6 +3376,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows" version = "0.51.1" diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index de8dd7bb9..85f18cc13 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -11,14 +11,24 @@ authors = { workspace = true } license = { workspace = true } [dependencies] -gourgeist = { version = "0.0.4", path = "../gourgeist" } -pep508_rs = { version = "0.2.3", path = "../pep508-rs" } +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-client = { path = "../puffin-client" } +puffin-installer = { path = "../puffin-installer" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } +puffin-resolver = { path = "../puffin-resolver" } +puffin-workspace = { path = "../puffin-workspace" } anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +directories = { workspace = true } flate2 = { workspace = true } fs-err = { workspace = true } indoc = { workspace = true } +itertools = { workspace = true } owo-colors = { workspace = true } pyproject-toml = { workspace = true } serde = { workspace = true } @@ -26,6 +36,7 @@ serde_json = { workspace = true } tar = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index 66e84eb53..fa0fd0fbf 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -8,15 +8,24 @@ use fs_err as fs; use fs_err::{DirEntry, File}; use gourgeist::{InterpreterInfo, Venv}; use indoc::formatdoc; +use itertools::{Either, Itertools}; use pep508_rs::Requirement; +use platform_host::Platform; +use platform_tags::Tags; +use puffin_client::PypiClientBuilder; +use puffin_installer::{Downloader, LocalDistribution, LocalIndex, RemoteDistribution, Unzipper}; +use puffin_interpreter::PythonExecutable; +use puffin_package::package_name::PackageName; +use puffin_resolver::WheelFinder; use pyproject_toml::PyProjectToml; use std::io; use std::io::BufRead; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; +use std::str::FromStr; use tar::Archive; -use tempfile::TempDir; +use tempfile::{tempdir, TempDir}; use thiserror::Error; use tracing::{debug, instrument}; use zip::ZipArchive; @@ -101,12 +110,13 @@ pub struct SourceDistributionBuilder { impl SourceDistributionBuilder { /// Extract the source distribution and create a venv with the required packages - pub fn setup( + pub async fn setup( sdist: &Path, base_python: &Path, interpreter_info: &InterpreterInfo, + cache: Option<&Path>, ) -> Result { - let temp_dir = TempDir::new()?; + let temp_dir = tempdir()?; // TODO(konstin): Parse and verify filenames debug!("Unpacking for build {}", sdist.display()); @@ -141,7 +151,9 @@ impl SourceDistributionBuilder { base_python, interpreter_info, pep517_backend, - )? + cache, + ) + .await? } else { if !source_tree.join("setup.py").is_file() { return Err(Error::InvalidSourceDistribution( @@ -149,12 +161,22 @@ impl SourceDistributionBuilder { .to_string(), )); } - gourgeist::create_venv( + let venv = gourgeist::create_venv( temp_dir.path().join("venv"), base_python, interpreter_info, - false, - )? + true, + )?; + // TODO: Resolve those once globally and cache per puffin invocation + let requirements = [ + Requirement::from_str("wheel").unwrap(), + Requirement::from_str("setuptools").unwrap(), + Requirement::from_str("pip").unwrap(), + ]; + resolve_and_install(venv.as_std_path(), &requirements, cache) + .await + .map_err(Error::RequirementsInstall)?; + venv }; Ok(Self { @@ -323,17 +345,22 @@ fn escape_path_for_python(path: &Path) -> String { } /// Not a method because we call it before the builder is completely initialized -fn create_pep517_build_environment( +async fn create_pep517_build_environment( root: &Path, source_tree: &Path, base_python: &Path, data: &InterpreterInfo, pep517_backend: &Pep517Backend, + cache: Option<&Path>, ) -> Result { - // TODO(konstin): Create bare venvs when we don't need pip anymore - let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, false)?; - resolve_and_install(venv.deref().as_std_path(), &pep517_backend.requirements) - .map_err(Error::RequirementsInstall)?; + let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, true)?; + resolve_and_install( + venv.deref().as_std_path(), + &pep517_backend.requirements, + cache, + ) + .await + .map_err(Error::RequirementsInstall)?; debug!( "Calling `{}.get_requires_for_build_wheel()`", @@ -393,30 +420,60 @@ fn create_pep517_build_environment( .cloned() .chain(extra_requires) .collect(); - resolve_and_install(&*venv, &requirements).map_err(Error::RequirementsInstall)?; + resolve_and_install(&*venv, &requirements, cache) + .await + .map_err(Error::RequirementsInstall)?; } Ok(venv) } #[instrument(skip_all)] -fn resolve_and_install(venv: impl AsRef, requirements: &[Requirement]) -> anyhow::Result<()> { - debug!("Calling pip to install build dependencies"); - let python = Venv::new(venv.as_ref())?.python_interpreter(); - // No error handling because we want have to replace this with the real resolver and installer - // anyway. - let installation = Command::new(python) - .args(["-m", "pip", "install"]) - .args( - requirements - .iter() - .map(ToString::to_string) - .collect::>(), - ) - .output() - .context("pip install failed")?; - if !installation.status.success() { - anyhow::bail!("Installation failed :(") - } +async fn resolve_and_install( + venv: impl AsRef, + requirements: &[Requirement], + cache: Option<&Path>, +) -> anyhow::Result<()> { + debug!("Installing {} build requirements", requirements.len()); + + let local_index = if let Some(cache) = cache { + LocalIndex::from_directory(cache).await? + } else { + LocalIndex::default() + }; + let (cached, uncached): (Vec, Vec) = + requirements.iter().partition_map(|requirement| { + let package = PackageName::normalize(&requirement.name); + if let Some(distribution) = local_index + .get(&package) + .filter(|dist| requirement.is_satisfied_by(dist.version())) + { + Either::Left(distribution.clone()) + } else { + Either::Right(requirement.clone()) + } + }); + + let client = PypiClientBuilder::default().cache(cache).build(); + + let platform = Platform::current()?; + let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?; + let tags = Tags::from_env(python.platform(), python.simple_version())?; + let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?; + let uncached = resolution + .into_files() + .map(RemoteDistribution::from_file) + .collect::>>()?; + let staging = tempdir()?; + let downloads = Downloader::new(&client, cache) + .download(&uncached, cache.unwrap_or(staging.path())) + .await?; + let unzips = Unzipper::default() + .download(downloads, cache.unwrap_or(staging.path())) + .await + .context("Failed to download and unpack wheels")?; + let wheels = unzips.into_iter().chain(cached).collect::>(); + puffin_installer::Installer::new(&python).install(&wheels)?; + Ok(()) } diff --git a/crates/puffin-build/src/main.rs b/crates/puffin-build/src/main.rs index 366114f2e..c18ffb4f9 100644 --- a/crates/puffin-build/src/main.rs +++ b/crates/puffin-build/src/main.rs @@ -2,6 +2,7 @@ use anyhow::Context; use clap::Parser; +use directories::ProjectDirs; use fs_err as fs; use owo_colors::OwoColorize; use puffin_build::{Error, SourceDistributionBuilder}; @@ -27,7 +28,7 @@ struct Args { sdist: PathBuf, } -fn run() -> anyhow::Result<()> { +async fn run() -> anyhow::Result<()> { let args = Args::parse(); let wheel_dir = if let Some(wheel_dir) = args.wheels { fs::create_dir_all(&wheel_dir).context("Invalid wheel directory")?; @@ -36,6 +37,9 @@ fn run() -> anyhow::Result<()> { env::current_dir()? }; + let dirs = ProjectDirs::from("", "", "puffin"); + let cache = dirs.as_ref().map(ProjectDirs::cache_dir); + // TODO: That's no way to deal with paths in PATH let base_python = which::which(args.python.unwrap_or("python3".into())).map_err(|err| { Error::IO(io::Error::new( @@ -45,20 +49,23 @@ fn run() -> anyhow::Result<()> { })?; let interpreter_info = gourgeist::get_interpreter_info(&base_python)?; - let builder = SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info)?; + let builder = + SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info, cache) + .await?; let wheel = builder.build(&wheel_dir)?; println!("Wheel built to {}", wheel.display()); Ok(()) } -fn main() -> ExitCode { +#[tokio::main] +async fn main() -> ExitCode { tracing_subscriber::registry() .with(fmt::layer().with_span_events(FmtSpan::CLOSE)) .with(EnvFilter::from_default_env()) .init(); let start = Instant::now(); - let result = run(); + let result = run().await; debug!("Took {}ms", start.elapsed().as_millis()); if let Err(err) = result { eprintln!("{}", "puffin-build failed".red().bold()); diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs index 0ee0af569..9e2f36a21 100644 --- a/crates/puffin-cli/src/commands/compile.rs +++ b/crates/puffin-cli/src/commands/compile.rs @@ -53,13 +53,7 @@ pub(crate) async fn compile( let tags = Tags::from_env(python.platform(), python.simple_version())?; // Instantiate a client. - let client = { - let mut pypi_client = PypiClientBuilder::default(); - if let Some(cache) = cache { - pypi_client = pypi_client.cache(cache); - } - pypi_client.build() - }; + let client = PypiClientBuilder::default().cache(cache).build(); // Resolve the dependencies. let resolver = puffin_resolver::Resolver::new(requirements, markers, &tags, &client); diff --git a/crates/puffin-cli/src/commands/sync.rs b/crates/puffin-cli/src/commands/sync.rs index f72ea05f3..46ee10b0d 100644 --- a/crates/puffin-cli/src/commands/sync.rs +++ b/crates/puffin-cli/src/commands/sync.rs @@ -1,16 +1,17 @@ use std::fmt::Write; use std::path::Path; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use bitflags::bitflags; use itertools::{Either, Itertools}; use owo_colors::OwoColorize; +use pep508_rs::Requirement; use tracing::debug; use platform_host::Platform; use platform_tags::Tags; use puffin_client::PypiClientBuilder; -use puffin_installer::{LocalIndex, RemoteDistribution}; +use puffin_installer::{LocalDistribution, LocalIndex, RemoteDistribution}; use puffin_interpreter::{PythonExecutable, SitePackages}; use puffin_package::package_name::PackageName; use puffin_package::requirements_txt::RequirementsTxt; @@ -36,10 +37,11 @@ pub(crate) async fn sync( flags: SyncFlags, mut printer: Printer, ) -> Result { - let start = std::time::Instant::now(); - // Read the `requirements.txt` from disk. let requirements_txt = RequirementsTxt::parse(src, std::env::current_dir()?)?; + if !requirements_txt.constraints.is_empty() { + bail!("Constraints in requirements.txt are not supported"); + } let requirements = requirements_txt .requirements .into_iter() @@ -50,6 +52,18 @@ pub(crate) async fn sync( return Ok(ExitStatus::Success); } + sync_requirements(&requirements, cache, flags, printer).await +} + +/// Install a set of locked requirements into the current Python environment. +pub(crate) async fn sync_requirements( + requirements: &[Requirement], + cache: Option<&Path>, + flags: SyncFlags, + mut printer: Printer, +) -> Result { + let start = std::time::Instant::now(); + // Detect the current Python interpreter. let platform = Platform::current()?; let python = PythonExecutable::from_env(platform, cache)?; @@ -61,57 +75,9 @@ pub(crate) async fn sync( // Determine the current environment markers. let tags = Tags::from_env(python.platform(), python.simple_version())?; - // Index all the already-installed packages in site-packages. - let site_packages = if flags.intersects(SyncFlags::IGNORE_INSTALLED) { - SitePackages::default() - } else { - SitePackages::from_executable(&python).await? - }; - - // Index all the already-downloaded wheels in the cache. - let local_index = if let Some(cache) = cache { - LocalIndex::from_directory(cache).await? - } else { - LocalIndex::default() - }; - // Filter out any already-installed or already-cached packages. - let (cached, uncached): (Vec<_>, Vec<_>) = requirements - .iter() - .filter(|requirement| { - let package = PackageName::normalize(&requirement.name); - - // Filter out already-installed packages. - if let Some(dist_info) = site_packages.get(&package) { - debug!( - "Requirement already satisfied: {} ({})", - package, - dist_info.version() - ); - false - } else { - true - } - }) - .partition_map(|requirement| { - let package = PackageName::normalize(&requirement.name); - - // Identify any locally-available distributions that satisfy the requirement. - if let Some(distribution) = local_index - .get(&package) - .filter(|dist| requirement.is_satisfied_by(dist.version())) - { - debug!( - "Requirement already cached: {} ({})", - distribution.name(), - distribution.version() - ); - Either::Left(distribution.clone()) - } else { - debug!("Identified uncached requirement: {}", requirement); - Either::Right(requirement.clone()) - } - }); + let (cached, uncached) = + find_uncached_requirements(requirements, cache, flags, &python).await?; // Nothing to do. if uncached.is_empty() && cached.is_empty() { @@ -130,22 +96,14 @@ pub(crate) async fn sync( return Ok(ExitStatus::Success); } - let client = { - let mut pypi_client = PypiClientBuilder::default(); - if let Some(cache) = cache { - pypi_client = pypi_client.cache(cache); - } - pypi_client.build() - }; + let client = PypiClientBuilder::default().cache(cache).build(); // Resolve the dependencies. - let resolution = if uncached.is_empty() { - puffin_resolver::Resolution::default() - } else { - let wheel_finder = puffin_resolver::WheelFinder::new(&tags, &client) - .with_reporter(WheelFinderReporter::from(printer).with_length(uncached.len() as u64)); - let resolution = wheel_finder.resolve(&uncached).await?; + let wheel_finder = puffin_resolver::WheelFinder::new(&tags, &client) + .with_reporter(WheelFinderReporter::from(printer).with_length(uncached.len() as u64)); + let resolution = wheel_finder.resolve(&uncached).await?; + if !resolution.is_empty() { let s = if resolution.len() == 1 { "" } else { "s" }; writeln!( printer, @@ -157,9 +115,7 @@ pub(crate) async fn sync( ) .dimmed() )?; - - resolution - }; + } let start = std::time::Instant::now(); @@ -256,3 +212,73 @@ pub(crate) async fn sync( Ok(ExitStatus::Success) } + +async fn find_uncached_requirements( + requirements: &[Requirement], + cache: Option<&Path>, + flags: SyncFlags, + python: &PythonExecutable, +) -> Result<(Vec, Vec)> { + // Index all the already-installed packages in site-packages. + let site_packages = if flags.intersects(SyncFlags::IGNORE_INSTALLED) { + SitePackages::default() + } else { + SitePackages::from_executable(python).await? + }; + + // Index all the already-downloaded wheels in the cache. + let local_index = if let Some(cache) = cache { + LocalIndex::from_directory(cache).await? + } else { + LocalIndex::default() + }; + + Ok(split_uncached_requirements( + requirements, + &site_packages, + &local_index, + )) +} + +fn split_uncached_requirements( + requirements: &[Requirement], + site_packages: &SitePackages, + local_index: &LocalIndex, +) -> (Vec, Vec) { + requirements + .iter() + .filter(|requirement| { + let package = PackageName::normalize(&requirement.name); + + // Filter out already-installed packages. + if let Some(dist_info) = site_packages.get(&package) { + debug!( + "Requirement already satisfied: {} ({})", + package, + dist_info.version() + ); + false + } else { + true + } + }) + .partition_map(|requirement| { + let package = PackageName::normalize(&requirement.name); + + // Identify any locally-available distributions that satisfy the requirement. + if let Some(distribution) = local_index + .get(&package) + .filter(|dist| requirement.is_satisfied_by(dist.version())) + { + debug!( + "Requirement already cached: {} ({})", + distribution.name(), + distribution.version() + ); + Either::Left(distribution.clone()) + } else { + debug!("Identified uncached requirement: {}", requirement); + Either::Right(requirement.clone()) + } + }) +} diff --git a/crates/puffin-client/src/client.rs b/crates/puffin-client/src/client.rs index 6d0c14fc8..0a79507df 100644 --- a/crates/puffin-client/src/client.rs +++ b/crates/puffin-client/src/client.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions}; @@ -47,8 +47,11 @@ impl PypiClientBuilder { } #[must_use] - pub fn cache(mut self, cache: impl AsRef) -> Self { - self.cache = Some(PathBuf::from(cache.as_ref())); + pub fn cache(mut self, cache: Option) -> Self + where + T: Into, + { + self.cache = cache.map(Into::into); self } diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index 1c8c4f8c7..279f45e96 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -17,7 +17,7 @@ puffin-package = { path = "../puffin-package" } anyhow = { workspace = true } cacache = { workspace = true } -fs-err = { workspace = true } +fs-err = { workspace = true, features = ["tokio"] } serde_json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index fb6a9ff91..e021863c0 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -39,6 +39,19 @@ impl PythonExecutable { }) } + pub fn from_venv(platform: Platform, venv: &Path, cache: Option<&Path>) -> Result { + let platform = PythonPlatform::from(platform); + let executable = platform.venv_python(venv); + let markers = markers::detect_cached_markers(&executable, cache)?; + + Ok(Self { + platform, + venv: venv.to_path_buf(), + executable, + markers, + }) + } + /// Returns the path to the Python virtual environment. pub fn platform(&self) -> &Platform { &self.platform diff --git a/scripts/benchmarks/sync.sh b/scripts/benchmarks/sync.sh index ee93ab7bd..47d51b083 100755 --- a/scripts/benchmarks/sync.sh +++ b/scripts/benchmarks/sync.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash ### # Benchmark the installer against `pip`. @@ -16,8 +16,8 @@ TARGET=${1} # Installation with a cold cache. ### hyperfine --runs 20 --warmup 3 \ - --prepare "rm -rf .venv && virtualenv .venv" \ - "./target/release/puffin-cli sync ${TARGET} --ignore-installed --no-cache" \ + --prepare "virtualenv --clear .venv" \ + "./target/release/puffin sync ${TARGET} --ignore-installed --no-cache" \ --prepare "rm -rf /tmp/site-packages" \ "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-cache-dir --no-deps" @@ -25,8 +25,8 @@ hyperfine --runs 20 --warmup 3 \ # Installation with a warm cache, similar to blowing away and re-creating a virtual environment. ### hyperfine --runs 20 --warmup 3 \ - --prepare "rm -rf .venv && virtualenv .venv" \ - "./target/release/puffin-cli sync ${TARGET} --ignore-installed" \ + --prepare "virtualenv --clear .venv" \ + "./target/release/puffin sync ${TARGET} --ignore-installed" \ --prepare "rm -rf /tmp/site-packages" \ "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-deps" @@ -34,6 +34,6 @@ hyperfine --runs 20 --warmup 3 \ # Installation with all dependencies already installed (no-op). ### hyperfine --runs 20 --warmup 3 \ - --setup "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \ - "./target/release/puffin-cli sync ${TARGET}" \ + --setup "virtualenv --clear .venv && source .venv/bin/activate" \ + "./target/release/puffin sync ${TARGET}" \ "pip install -r ${TARGET} --no-deps"