mirror of
https://github.com/astral-sh/uv.git
synced 2025-07-07 13:25:00 +00:00
Install source distribution requirements with puffin itself instead of pip (#122)
This is also a lot faster. Unfortunately it copies a lot of code from the sync cli since the `Printer` is private. The first commit are some refactorings i made when i thought about how i could reuse the existing code.
This commit is contained in:
konsti
GitHub
parent
7bc42ca2ce
commit
8cc4fe0d44
10 changed files with 306 additions and 193 deletions
138
Cargo.lock
generated
138
Cargo.lock
generated
|
|
@ -138,9 +138,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
|
|||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.4"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2"
|
||||
checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
|
|
@ -152,9 +152,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.74"
|
||||
version = "0.1.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
|
||||
checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -220,9 +220,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.4.1"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
|
||||
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
|
|
@ -571,12 +571,9 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
|
|||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.3.9"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
|
|
@ -1083,16 +1080,16 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.58"
|
||||
version = "0.1.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
|
||||
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
"windows 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1309,9 +1306,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
|
|||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.11"
|
||||
version = "0.4.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
|
||||
checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
|
|
@ -1509,7 +1506,7 @@ version = "0.10.57"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"bitflags 2.4.0",
|
||||
"cfg-if 1.0.0",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
|
|
@ -1583,7 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core 0.9.9",
|
||||
"parking_lot_core 0.9.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1602,13 +1599,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.9"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
|
||||
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
"redox_syscall 0.4.1",
|
||||
"redox_syscall 0.3.5",
|
||||
"smallvec",
|
||||
"windows-targets 0.48.5",
|
||||
]
|
||||
|
|
@ -1741,12 +1738,6 @@ version = "1.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -1815,18 +1806,29 @@ version = "0.0.1"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"directories",
|
||||
"flate2",
|
||||
"fs-err",
|
||||
"gourgeist",
|
||||
"indoc 2.0.4",
|
||||
"itertools",
|
||||
"owo-colors",
|
||||
"pep508_rs",
|
||||
"platform-host",
|
||||
"platform-tags",
|
||||
"puffin-client",
|
||||
"puffin-installer",
|
||||
"puffin-interpreter",
|
||||
"puffin-package",
|
||||
"puffin-resolver",
|
||||
"puffin-workspace",
|
||||
"pyproject-toml",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"toml 0.8.2",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
|
@ -1839,7 +1841,7 @@ name = "puffin-cli"
|
|||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.4.1",
|
||||
"bitflags 2.4.0",
|
||||
"cacache",
|
||||
"clap",
|
||||
"directories",
|
||||
|
|
@ -1956,7 +1958,7 @@ name = "puffin-resolver"
|
|||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.4.1",
|
||||
"bitflags 2.4.0",
|
||||
"futures",
|
||||
"once_cell",
|
||||
"pep440_rs 0.3.12",
|
||||
|
|
@ -2026,9 +2028,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "pyo3-log"
|
||||
version = "0.8.4"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c09c2b349b6538d8a73d436ca606dab6ce0aaab4dad9e6b7bdd57a4f556c3bc3"
|
||||
checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"log",
|
||||
|
|
@ -2163,15 +2165,6 @@ dependencies = [
|
|||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
|
|
@ -2185,25 +2178,25 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "reflink-copy"
|
||||
version = "0.1.10"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c9bd37fcf997c2d9ec7ebdff893c396677664164cf72105b063ac4a483702d3"
|
||||
checksum = "d7e3e017e993f86feeddf8a7fb609ca49f89082309e328e27aefd4a25bb317a4"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"ioctl-sys",
|
||||
"windows",
|
||||
"windows 0.51.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.2"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
|
||||
checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata 0.4.3",
|
||||
"regex-syntax 0.8.2",
|
||||
"regex-automata 0.4.1",
|
||||
"regex-syntax 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2217,13 +2210,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.3"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
|
||||
checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax 0.8.2",
|
||||
"regex-syntax 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2234,9 +2227,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
|||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
|
||||
checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
|
|
@ -2357,11 +2350,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.19"
|
||||
version = "0.38.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
|
||||
checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"bitflags 2.4.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
|
|
@ -2455,18 +2448,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.189"
|
||||
version = "1.0.188"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
|
||||
checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.189"
|
||||
version = "1.0.188"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
|
||||
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2837,13 +2830,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.30"
|
||||
version = "0.3.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
|
||||
checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"powerfmt",
|
||||
"serde",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
|
|
@ -3019,10 +3011,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
|
|||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.39"
|
||||
version = "0.1.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9"
|
||||
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
|
|
@ -3031,9 +3024,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.27"
|
||||
version = "0.1.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
|
||||
checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -3042,9 +3035,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.32"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
|
||||
checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
|
|
@ -3383,6 +3376,15 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
|
||||
dependencies = [
|
||||
"windows-targets 0.48.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.51.1"
|
||||
|
|
|
|||
|
|
@ -11,14 +11,24 @@ authors = { workspace = true }
|
|||
license = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
gourgeist = { version = "0.0.4", path = "../gourgeist" }
|
||||
pep508_rs = { version = "0.2.3", path = "../pep508-rs" }
|
||||
gourgeist = { path = "../gourgeist" }
|
||||
pep508_rs = { path = "../pep508-rs" }
|
||||
platform-host = { path = "../platform-host" }
|
||||
platform-tags = { path = "../platform-tags" }
|
||||
puffin-client = { path = "../puffin-client" }
|
||||
puffin-installer = { path = "../puffin-installer" }
|
||||
puffin-interpreter = { path = "../puffin-interpreter" }
|
||||
puffin-package = { path = "../puffin-package" }
|
||||
puffin-resolver = { path = "../puffin-resolver" }
|
||||
puffin-workspace = { path = "../puffin-workspace" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
directories = { workspace = true }
|
||||
flate2 = { workspace = true }
|
||||
fs-err = { workspace = true }
|
||||
indoc = { workspace = true }
|
||||
itertools = { workspace = true }
|
||||
owo-colors = { workspace = true }
|
||||
pyproject-toml = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
|
@ -26,6 +36,7 @@ serde_json = { workspace = true }
|
|||
tar = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -8,15 +8,24 @@ use fs_err as fs;
|
|||
use fs_err::{DirEntry, File};
|
||||
use gourgeist::{InterpreterInfo, Venv};
|
||||
use indoc::formatdoc;
|
||||
use itertools::{Either, Itertools};
|
||||
use pep508_rs::Requirement;
|
||||
use platform_host::Platform;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_installer::{Downloader, LocalDistribution, LocalIndex, RemoteDistribution, Unzipper};
|
||||
use puffin_interpreter::PythonExecutable;
|
||||
use puffin_package::package_name::PackageName;
|
||||
use puffin_resolver::WheelFinder;
|
||||
use pyproject_toml::PyProjectToml;
|
||||
use std::io;
|
||||
use std::io::BufRead;
|
||||
use std::ops::Deref;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Output};
|
||||
use std::str::FromStr;
|
||||
use tar::Archive;
|
||||
use tempfile::TempDir;
|
||||
use tempfile::{tempdir, TempDir};
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, instrument};
|
||||
use zip::ZipArchive;
|
||||
|
|
@ -101,12 +110,13 @@ pub struct SourceDistributionBuilder {
|
|||
|
||||
impl SourceDistributionBuilder {
|
||||
/// Extract the source distribution and create a venv with the required packages
|
||||
pub fn setup(
|
||||
pub async fn setup(
|
||||
sdist: &Path,
|
||||
base_python: &Path,
|
||||
interpreter_info: &InterpreterInfo,
|
||||
cache: Option<&Path>,
|
||||
) -> Result<SourceDistributionBuilder, Error> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let temp_dir = tempdir()?;
|
||||
|
||||
// TODO(konstin): Parse and verify filenames
|
||||
debug!("Unpacking for build {}", sdist.display());
|
||||
|
|
@ -141,7 +151,9 @@ impl SourceDistributionBuilder {
|
|||
base_python,
|
||||
interpreter_info,
|
||||
pep517_backend,
|
||||
)?
|
||||
cache,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
if !source_tree.join("setup.py").is_file() {
|
||||
return Err(Error::InvalidSourceDistribution(
|
||||
|
|
@ -149,12 +161,22 @@ impl SourceDistributionBuilder {
|
|||
.to_string(),
|
||||
));
|
||||
}
|
||||
gourgeist::create_venv(
|
||||
let venv = gourgeist::create_venv(
|
||||
temp_dir.path().join("venv"),
|
||||
base_python,
|
||||
interpreter_info,
|
||||
false,
|
||||
)?
|
||||
true,
|
||||
)?;
|
||||
// TODO: Resolve those once globally and cache per puffin invocation
|
||||
let requirements = [
|
||||
Requirement::from_str("wheel").unwrap(),
|
||||
Requirement::from_str("setuptools").unwrap(),
|
||||
Requirement::from_str("pip").unwrap(),
|
||||
];
|
||||
resolve_and_install(venv.as_std_path(), &requirements, cache)
|
||||
.await
|
||||
.map_err(Error::RequirementsInstall)?;
|
||||
venv
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
|
|
@ -323,17 +345,22 @@ fn escape_path_for_python(path: &Path) -> String {
|
|||
}
|
||||
|
||||
/// Not a method because we call it before the builder is completely initialized
|
||||
fn create_pep517_build_environment(
|
||||
async fn create_pep517_build_environment(
|
||||
root: &Path,
|
||||
source_tree: &Path,
|
||||
base_python: &Path,
|
||||
data: &InterpreterInfo,
|
||||
pep517_backend: &Pep517Backend,
|
||||
cache: Option<&Path>,
|
||||
) -> Result<Venv, Error> {
|
||||
// TODO(konstin): Create bare venvs when we don't need pip anymore
|
||||
let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, false)?;
|
||||
resolve_and_install(venv.deref().as_std_path(), &pep517_backend.requirements)
|
||||
.map_err(Error::RequirementsInstall)?;
|
||||
let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, true)?;
|
||||
resolve_and_install(
|
||||
venv.deref().as_std_path(),
|
||||
&pep517_backend.requirements,
|
||||
cache,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::RequirementsInstall)?;
|
||||
|
||||
debug!(
|
||||
"Calling `{}.get_requires_for_build_wheel()`",
|
||||
|
|
@ -393,30 +420,60 @@ fn create_pep517_build_environment(
|
|||
.cloned()
|
||||
.chain(extra_requires)
|
||||
.collect();
|
||||
resolve_and_install(&*venv, &requirements).map_err(Error::RequirementsInstall)?;
|
||||
resolve_and_install(&*venv, &requirements, cache)
|
||||
.await
|
||||
.map_err(Error::RequirementsInstall)?;
|
||||
}
|
||||
Ok(venv)
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
fn resolve_and_install(venv: impl AsRef<Path>, requirements: &[Requirement]) -> anyhow::Result<()> {
|
||||
debug!("Calling pip to install build dependencies");
|
||||
let python = Venv::new(venv.as_ref())?.python_interpreter();
|
||||
// No error handling because we want have to replace this with the real resolver and installer
|
||||
// anyway.
|
||||
let installation = Command::new(python)
|
||||
.args(["-m", "pip", "install"])
|
||||
.args(
|
||||
requirements
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<String>>(),
|
||||
)
|
||||
.output()
|
||||
.context("pip install failed")?;
|
||||
if !installation.status.success() {
|
||||
anyhow::bail!("Installation failed :(")
|
||||
}
|
||||
async fn resolve_and_install(
|
||||
venv: impl AsRef<Path>,
|
||||
requirements: &[Requirement],
|
||||
cache: Option<&Path>,
|
||||
) -> anyhow::Result<()> {
|
||||
debug!("Installing {} build requirements", requirements.len());
|
||||
|
||||
let local_index = if let Some(cache) = cache {
|
||||
LocalIndex::from_directory(cache).await?
|
||||
} else {
|
||||
LocalIndex::default()
|
||||
};
|
||||
let (cached, uncached): (Vec<LocalDistribution>, Vec<Requirement>) =
|
||||
requirements.iter().partition_map(|requirement| {
|
||||
let package = PackageName::normalize(&requirement.name);
|
||||
if let Some(distribution) = local_index
|
||||
.get(&package)
|
||||
.filter(|dist| requirement.is_satisfied_by(dist.version()))
|
||||
{
|
||||
Either::Left(distribution.clone())
|
||||
} else {
|
||||
Either::Right(requirement.clone())
|
||||
}
|
||||
});
|
||||
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
|
||||
let platform = Platform::current()?;
|
||||
let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?;
|
||||
let tags = Tags::from_env(python.platform(), python.simple_version())?;
|
||||
let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?;
|
||||
let uncached = resolution
|
||||
.into_files()
|
||||
.map(RemoteDistribution::from_file)
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
let staging = tempdir()?;
|
||||
let downloads = Downloader::new(&client, cache)
|
||||
.download(&uncached, cache.unwrap_or(staging.path()))
|
||||
.await?;
|
||||
let unzips = Unzipper::default()
|
||||
.download(downloads, cache.unwrap_or(staging.path()))
|
||||
.await
|
||||
.context("Failed to download and unpack wheels")?;
|
||||
let wheels = unzips.into_iter().chain(cached).collect::<Vec<_>>();
|
||||
puffin_installer::Installer::new(&python).install(&wheels)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
use anyhow::Context;
|
||||
use clap::Parser;
|
||||
use directories::ProjectDirs;
|
||||
use fs_err as fs;
|
||||
use owo_colors::OwoColorize;
|
||||
use puffin_build::{Error, SourceDistributionBuilder};
|
||||
|
|
@ -27,7 +28,7 @@ struct Args {
|
|||
sdist: PathBuf,
|
||||
}
|
||||
|
||||
fn run() -> anyhow::Result<()> {
|
||||
async fn run() -> anyhow::Result<()> {
|
||||
let args = Args::parse();
|
||||
let wheel_dir = if let Some(wheel_dir) = args.wheels {
|
||||
fs::create_dir_all(&wheel_dir).context("Invalid wheel directory")?;
|
||||
|
|
@ -36,6 +37,9 @@ fn run() -> anyhow::Result<()> {
|
|||
env::current_dir()?
|
||||
};
|
||||
|
||||
let dirs = ProjectDirs::from("", "", "puffin");
|
||||
let cache = dirs.as_ref().map(ProjectDirs::cache_dir);
|
||||
|
||||
// TODO: That's no way to deal with paths in PATH
|
||||
let base_python = which::which(args.python.unwrap_or("python3".into())).map_err(|err| {
|
||||
Error::IO(io::Error::new(
|
||||
|
|
@ -45,20 +49,23 @@ fn run() -> anyhow::Result<()> {
|
|||
})?;
|
||||
let interpreter_info = gourgeist::get_interpreter_info(&base_python)?;
|
||||
|
||||
let builder = SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info)?;
|
||||
let builder =
|
||||
SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info, cache)
|
||||
.await?;
|
||||
let wheel = builder.build(&wheel_dir)?;
|
||||
println!("Wheel built to {}", wheel.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
#[tokio::main]
|
||||
async fn main() -> ExitCode {
|
||||
tracing_subscriber::registry()
|
||||
.with(fmt::layer().with_span_events(FmtSpan::CLOSE))
|
||||
.with(EnvFilter::from_default_env())
|
||||
.init();
|
||||
|
||||
let start = Instant::now();
|
||||
let result = run();
|
||||
let result = run().await;
|
||||
debug!("Took {}ms", start.elapsed().as_millis());
|
||||
if let Err(err) = result {
|
||||
eprintln!("{}", "puffin-build failed".red().bold());
|
||||
|
|
|
|||
|
|
@ -53,13 +53,7 @@ pub(crate) async fn compile(
|
|||
let tags = Tags::from_env(python.platform(), python.simple_version())?;
|
||||
|
||||
// Instantiate a client.
|
||||
let client = {
|
||||
let mut pypi_client = PypiClientBuilder::default();
|
||||
if let Some(cache) = cache {
|
||||
pypi_client = pypi_client.cache(cache);
|
||||
}
|
||||
pypi_client.build()
|
||||
};
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
|
||||
// Resolve the dependencies.
|
||||
let resolver = puffin_resolver::Resolver::new(requirements, markers, &tags, &client);
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
use std::fmt::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use anyhow::{bail, Context, Result};
|
||||
use bitflags::bitflags;
|
||||
use itertools::{Either, Itertools};
|
||||
use owo_colors::OwoColorize;
|
||||
use pep508_rs::Requirement;
|
||||
use tracing::debug;
|
||||
|
||||
use platform_host::Platform;
|
||||
use platform_tags::Tags;
|
||||
use puffin_client::PypiClientBuilder;
|
||||
use puffin_installer::{LocalIndex, RemoteDistribution};
|
||||
use puffin_installer::{LocalDistribution, LocalIndex, RemoteDistribution};
|
||||
use puffin_interpreter::{PythonExecutable, SitePackages};
|
||||
use puffin_package::package_name::PackageName;
|
||||
use puffin_package::requirements_txt::RequirementsTxt;
|
||||
|
|
@ -36,10 +37,11 @@ pub(crate) async fn sync(
|
|||
flags: SyncFlags,
|
||||
mut printer: Printer,
|
||||
) -> Result<ExitStatus> {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Read the `requirements.txt` from disk.
|
||||
let requirements_txt = RequirementsTxt::parse(src, std::env::current_dir()?)?;
|
||||
if !requirements_txt.constraints.is_empty() {
|
||||
bail!("Constraints in requirements.txt are not supported");
|
||||
}
|
||||
let requirements = requirements_txt
|
||||
.requirements
|
||||
.into_iter()
|
||||
|
|
@ -50,6 +52,18 @@ pub(crate) async fn sync(
|
|||
return Ok(ExitStatus::Success);
|
||||
}
|
||||
|
||||
sync_requirements(&requirements, cache, flags, printer).await
|
||||
}
|
||||
|
||||
/// Install a set of locked requirements into the current Python environment.
|
||||
pub(crate) async fn sync_requirements(
|
||||
requirements: &[Requirement],
|
||||
cache: Option<&Path>,
|
||||
flags: SyncFlags,
|
||||
mut printer: Printer,
|
||||
) -> Result<ExitStatus> {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Detect the current Python interpreter.
|
||||
let platform = Platform::current()?;
|
||||
let python = PythonExecutable::from_env(platform, cache)?;
|
||||
|
|
@ -61,57 +75,9 @@ pub(crate) async fn sync(
|
|||
// Determine the current environment markers.
|
||||
let tags = Tags::from_env(python.platform(), python.simple_version())?;
|
||||
|
||||
// Index all the already-installed packages in site-packages.
|
||||
let site_packages = if flags.intersects(SyncFlags::IGNORE_INSTALLED) {
|
||||
SitePackages::default()
|
||||
} else {
|
||||
SitePackages::from_executable(&python).await?
|
||||
};
|
||||
|
||||
// Index all the already-downloaded wheels in the cache.
|
||||
let local_index = if let Some(cache) = cache {
|
||||
LocalIndex::from_directory(cache).await?
|
||||
} else {
|
||||
LocalIndex::default()
|
||||
};
|
||||
|
||||
// Filter out any already-installed or already-cached packages.
|
||||
let (cached, uncached): (Vec<_>, Vec<_>) = requirements
|
||||
.iter()
|
||||
.filter(|requirement| {
|
||||
let package = PackageName::normalize(&requirement.name);
|
||||
|
||||
// Filter out already-installed packages.
|
||||
if let Some(dist_info) = site_packages.get(&package) {
|
||||
debug!(
|
||||
"Requirement already satisfied: {} ({})",
|
||||
package,
|
||||
dist_info.version()
|
||||
);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
})
|
||||
.partition_map(|requirement| {
|
||||
let package = PackageName::normalize(&requirement.name);
|
||||
|
||||
// Identify any locally-available distributions that satisfy the requirement.
|
||||
if let Some(distribution) = local_index
|
||||
.get(&package)
|
||||
.filter(|dist| requirement.is_satisfied_by(dist.version()))
|
||||
{
|
||||
debug!(
|
||||
"Requirement already cached: {} ({})",
|
||||
distribution.name(),
|
||||
distribution.version()
|
||||
);
|
||||
Either::Left(distribution.clone())
|
||||
} else {
|
||||
debug!("Identified uncached requirement: {}", requirement);
|
||||
Either::Right(requirement.clone())
|
||||
}
|
||||
});
|
||||
let (cached, uncached) =
|
||||
find_uncached_requirements(requirements, cache, flags, &python).await?;
|
||||
|
||||
// Nothing to do.
|
||||
if uncached.is_empty() && cached.is_empty() {
|
||||
|
|
@ -130,22 +96,14 @@ pub(crate) async fn sync(
|
|||
return Ok(ExitStatus::Success);
|
||||
}
|
||||
|
||||
let client = {
|
||||
let mut pypi_client = PypiClientBuilder::default();
|
||||
if let Some(cache) = cache {
|
||||
pypi_client = pypi_client.cache(cache);
|
||||
}
|
||||
pypi_client.build()
|
||||
};
|
||||
let client = PypiClientBuilder::default().cache(cache).build();
|
||||
|
||||
// Resolve the dependencies.
|
||||
let resolution = if uncached.is_empty() {
|
||||
puffin_resolver::Resolution::default()
|
||||
} else {
|
||||
let wheel_finder = puffin_resolver::WheelFinder::new(&tags, &client)
|
||||
.with_reporter(WheelFinderReporter::from(printer).with_length(uncached.len() as u64));
|
||||
let resolution = wheel_finder.resolve(&uncached).await?;
|
||||
let wheel_finder = puffin_resolver::WheelFinder::new(&tags, &client)
|
||||
.with_reporter(WheelFinderReporter::from(printer).with_length(uncached.len() as u64));
|
||||
let resolution = wheel_finder.resolve(&uncached).await?;
|
||||
|
||||
if !resolution.is_empty() {
|
||||
let s = if resolution.len() == 1 { "" } else { "s" };
|
||||
writeln!(
|
||||
printer,
|
||||
|
|
@ -157,9 +115,7 @@ pub(crate) async fn sync(
|
|||
)
|
||||
.dimmed()
|
||||
)?;
|
||||
|
||||
resolution
|
||||
};
|
||||
}
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
|
|
@ -256,3 +212,73 @@ pub(crate) async fn sync(
|
|||
|
||||
Ok(ExitStatus::Success)
|
||||
}
|
||||
|
||||
async fn find_uncached_requirements(
|
||||
requirements: &[Requirement],
|
||||
cache: Option<&Path>,
|
||||
flags: SyncFlags,
|
||||
python: &PythonExecutable,
|
||||
) -> Result<(Vec<LocalDistribution>, Vec<Requirement>)> {
|
||||
// Index all the already-installed packages in site-packages.
|
||||
let site_packages = if flags.intersects(SyncFlags::IGNORE_INSTALLED) {
|
||||
SitePackages::default()
|
||||
} else {
|
||||
SitePackages::from_executable(python).await?
|
||||
};
|
||||
|
||||
// Index all the already-downloaded wheels in the cache.
|
||||
let local_index = if let Some(cache) = cache {
|
||||
LocalIndex::from_directory(cache).await?
|
||||
} else {
|
||||
LocalIndex::default()
|
||||
};
|
||||
|
||||
Ok(split_uncached_requirements(
|
||||
requirements,
|
||||
&site_packages,
|
||||
&local_index,
|
||||
))
|
||||
}
|
||||
|
||||
fn split_uncached_requirements(
|
||||
requirements: &[Requirement],
|
||||
site_packages: &SitePackages,
|
||||
local_index: &LocalIndex,
|
||||
) -> (Vec<LocalDistribution>, Vec<Requirement>) {
|
||||
requirements
|
||||
.iter()
|
||||
.filter(|requirement| {
|
||||
let package = PackageName::normalize(&requirement.name);
|
||||
|
||||
// Filter out already-installed packages.
|
||||
if let Some(dist_info) = site_packages.get(&package) {
|
||||
debug!(
|
||||
"Requirement already satisfied: {} ({})",
|
||||
package,
|
||||
dist_info.version()
|
||||
);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
})
|
||||
.partition_map(|requirement| {
|
||||
let package = PackageName::normalize(&requirement.name);
|
||||
|
||||
// Identify any locally-available distributions that satisfy the requirement.
|
||||
if let Some(distribution) = local_index
|
||||
.get(&package)
|
||||
.filter(|dist| requirement.is_satisfied_by(dist.version()))
|
||||
{
|
||||
debug!(
|
||||
"Requirement already cached: {} ({})",
|
||||
distribution.name(),
|
||||
distribution.version()
|
||||
);
|
||||
Either::Left(distribution.clone())
|
||||
} else {
|
||||
debug!("Identified uncached requirement: {}", requirement);
|
||||
Either::Right(requirement.clone())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions};
|
||||
|
|
@ -47,8 +47,11 @@ impl PypiClientBuilder {
|
|||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn cache(mut self, cache: impl AsRef<Path>) -> Self {
|
||||
self.cache = Some(PathBuf::from(cache.as_ref()));
|
||||
pub fn cache<T>(mut self, cache: Option<T>) -> Self
|
||||
where
|
||||
T: Into<PathBuf>,
|
||||
{
|
||||
self.cache = cache.map(Into::into);
|
||||
self
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ puffin-package = { path = "../puffin-package" }
|
|||
|
||||
anyhow = { workspace = true }
|
||||
cacache = { workspace = true }
|
||||
fs-err = { workspace = true }
|
||||
fs-err = { workspace = true, features = ["tokio"] }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -39,6 +39,19 @@ impl PythonExecutable {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn from_venv(platform: Platform, venv: &Path, cache: Option<&Path>) -> Result<Self> {
|
||||
let platform = PythonPlatform::from(platform);
|
||||
let executable = platform.venv_python(venv);
|
||||
let markers = markers::detect_cached_markers(&executable, cache)?;
|
||||
|
||||
Ok(Self {
|
||||
platform,
|
||||
venv: venv.to_path_buf(),
|
||||
executable,
|
||||
markers,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the path to the Python virtual environment.
|
||||
pub fn platform(&self) -> &Platform {
|
||||
&self.platform
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
###
|
||||
# Benchmark the installer against `pip`.
|
||||
|
|
@ -16,8 +16,8 @@ TARGET=${1}
|
|||
# Installation with a cold cache.
|
||||
###
|
||||
hyperfine --runs 20 --warmup 3 \
|
||||
--prepare "rm -rf .venv && virtualenv .venv" \
|
||||
"./target/release/puffin-cli sync ${TARGET} --ignore-installed --no-cache" \
|
||||
--prepare "virtualenv --clear .venv" \
|
||||
"./target/release/puffin sync ${TARGET} --ignore-installed --no-cache" \
|
||||
--prepare "rm -rf /tmp/site-packages" \
|
||||
"pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-cache-dir --no-deps"
|
||||
|
||||
|
|
@ -25,8 +25,8 @@ hyperfine --runs 20 --warmup 3 \
|
|||
# Installation with a warm cache, similar to blowing away and re-creating a virtual environment.
|
||||
###
|
||||
hyperfine --runs 20 --warmup 3 \
|
||||
--prepare "rm -rf .venv && virtualenv .venv" \
|
||||
"./target/release/puffin-cli sync ${TARGET} --ignore-installed" \
|
||||
--prepare "virtualenv --clear .venv" \
|
||||
"./target/release/puffin sync ${TARGET} --ignore-installed" \
|
||||
--prepare "rm -rf /tmp/site-packages" \
|
||||
"pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-deps"
|
||||
|
||||
|
|
@ -34,6 +34,6 @@ hyperfine --runs 20 --warmup 3 \
|
|||
# Installation with all dependencies already installed (no-op).
|
||||
###
|
||||
hyperfine --runs 20 --warmup 3 \
|
||||
--setup "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
|
||||
"./target/release/puffin-cli sync ${TARGET}" \
|
||||
--setup "virtualenv --clear .venv && source .venv/bin/activate" \
|
||||
"./target/release/puffin sync ${TARGET}" \
|
||||
"pip install -r ${TARGET} --no-deps"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue