From ff8e24a621e43cbc3665e6d7579c99f0f95d3ac7 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 6 Oct 2023 15:31:21 -0400 Subject: [PATCH] Move `puffin-installer` to its own crate (#23) --- crates/README.md | 6 +- crates/puffin-cli/Cargo.toml | 4 +- crates/puffin-cli/src/commands/compile.rs | 3 +- crates/puffin-cli/src/commands/install.rs | 46 ++------------- crates/puffin-installer/Cargo.toml | 25 +++++++++ crates/puffin-installer/src/lib.rs | 56 +++++++++++++++++++ .../Cargo.toml | 8 +-- .../src/lib.rs | 14 ++--- requirements.in | 6 +- 9 files changed, 105 insertions(+), 63 deletions(-) create mode 100644 crates/puffin-installer/Cargo.toml create mode 100644 crates/puffin-installer/src/lib.rs rename crates/{puffin-resolve => puffin-resolver}/Cargo.toml (72%) rename crates/{puffin-resolve => puffin-resolver}/src/lib.rs (96%) diff --git a/crates/README.md b/crates/README.md index 4b40f78e8..418de43d8 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,6 +8,10 @@ Command-line interface for the Puffin package manager. Client for interacting with PyPI-compatible HTTP APIs. +## [puffin-installer](./puffin-installer) + +Functionality for installing Python packages into a virtual environment. + ## [puffin-interpreter](./puffin-interpreter) Functionality for detecting and leveraging the current Python interpreter. @@ -20,6 +24,6 @@ Types and functionality for working with Python packages, e.g., parsing wheel fi Functionality for detecting the current platform (operating system, architecture, etc.). -## [puffin-resolve](./puffin-resolve) +## [puffin-resolver](./puffin-resolver) Functionality for resolving Python packages and their dependencies. diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index e164d5e3b..e62ec861d 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -5,10 +5,11 @@ edition = "2021" [dependencies] puffin-client = { path = "../puffin-client" } +puffin-installer = { path = "../puffin-installer" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-platform = { path = "../puffin-platform" } puffin-package = { path = "../puffin-package" } -puffin-resolve = { path = "../puffin-resolve" } +puffin-resolver = { path = "../puffin-resolver" } anyhow = { version = "1.0.75" } async-std = { version = "1.12.0", features = [ @@ -26,6 +27,5 @@ pep440_rs = { version = "0.3.12" } tracing = { version = "0.1.37" } tracing-tree = { version = "0.2.5" } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -rayon = { version = "1.8.0" } url = { version = "2.4.1" } tempfile = { version = "3.8.0" } diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs index 29f19b10d..0bcd3cc30 100644 --- a/crates/puffin-cli/src/commands/compile.rs +++ b/crates/puffin-cli/src/commands/compile.rs @@ -8,7 +8,6 @@ use puffin_client::PypiClientBuilder; use puffin_interpreter::PythonExecutable; use puffin_platform::tags::Tags; use puffin_platform::Platform; -use puffin_resolve::resolve; use crate::commands::ExitStatus; @@ -44,7 +43,7 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result) -> Result>(); + puffin_installer::install(&wheels, &python, &client).await?; Ok(ExitStatus::Success) } diff --git a/crates/puffin-installer/Cargo.toml b/crates/puffin-installer/Cargo.toml new file mode 100644 index 000000000..f1a1ec330 --- /dev/null +++ b/crates/puffin-installer/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "puffin-installer" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +puffin-client = { path = "../puffin-client" } +puffin-interpreter = { path = "../puffin-interpreter" } + +anyhow = { version = "1.0.75" } +async-std = { version = "1.12.0", features = [ + "attributes", + "tokio1", + "unstable", +] } +install-wheel-rs = { version = "0.0.1" } +tempfile = { version = "3.8.0" } +tracing = { version = "0.1.37" } +url = { version = "2.4.1" } diff --git a/crates/puffin-installer/src/lib.rs b/crates/puffin-installer/src/lib.rs new file mode 100644 index 000000000..61f1cf5b7 --- /dev/null +++ b/crates/puffin-installer/src/lib.rs @@ -0,0 +1,56 @@ +use std::str::FromStr; + +use anyhow::Result; +use install_wheel_rs::{install_wheel, InstallLocation}; +use url::Url; + +use puffin_client::{File, PypiClient}; +use puffin_interpreter::PythonExecutable; + +/// Install a set of wheels into a Python virtual environment. +pub async fn install( + wheels: &[File], + python: &PythonExecutable, + client: &PypiClient, +) -> Result<()> { + // Create a temporary directory, in which we'll store the wheels. + let tmp_dir = tempfile::tempdir()?; + + // Download each wheel. + // TODO(charlie): Store these in a content-addressed cache. + // TODO(charlie): Use channels to efficiently stream-and-install. + for wheel in wheels { + let url = Url::parse(&wheel.url)?; + let reader = client.stream_external(&url).await?; + + // TODO(charlie): Stream the unzip. + let mut writer = + async_std::fs::File::create(tmp_dir.path().join(&wheel.hashes.sha256)).await?; + async_std::io::copy(reader, &mut writer).await?; + } + + // Install each wheel. + // TODO(charlie): Use channels to efficiently stream-and-install. + let location = InstallLocation::Venv { + venv_base: python.venv().to_path_buf(), + python_version: python.simple_version(), + }; + let locked_dir = location.acquire_lock()?; + for wheel in wheels { + let path = tmp_dir.path().join(&wheel.hashes.sha256); + let filename = install_wheel_rs::WheelFilename::from_str(&wheel.filename)?; + + // TODO(charlie): Should this be async? + install_wheel( + &locked_dir, + std::fs::File::open(path)?, + filename, + false, + &[], + "", + python.executable(), + )?; + } + + Ok(()) +} diff --git a/crates/puffin-resolve/Cargo.toml b/crates/puffin-resolver/Cargo.toml similarity index 72% rename from crates/puffin-resolve/Cargo.toml rename to crates/puffin-resolver/Cargo.toml index e518112b3..23cc45680 100644 --- a/crates/puffin-resolve/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "puffin-resolve" +name = "puffin-resolver" version = "0.1.0" edition.workspace = true rust-version.workspace = true @@ -11,15 +11,9 @@ license.workspace = true [dependencies] puffin-client = { path = "../puffin-client" } -puffin-interpreter = { path = "../puffin-interpreter" } puffin-platform = { path = "../puffin-platform" } puffin-package = { path = "../puffin-package" } -async-std = { version = "1.12.0", features = [ - "attributes", - "tokio1", - "unstable", -] } pep440_rs = "0.3.12" futures = "0.3.28" anyhow = "1.0.75" diff --git a/crates/puffin-resolve/src/lib.rs b/crates/puffin-resolver/src/lib.rs similarity index 96% rename from crates/puffin-resolve/src/lib.rs rename to crates/puffin-resolver/src/lib.rs index 25895f9cb..453a75031 100644 --- a/crates/puffin-resolve/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -19,9 +19,15 @@ use puffin_platform::tags::Tags; pub struct Resolution(HashMap); impl Resolution { + /// Iterate over the pinned packages in this resolution. pub fn iter(&self) -> impl Iterator { self.0.iter() } + + /// Iterate over the wheels in this resolution. + pub fn into_files(self) -> impl Iterator { + self.0.into_values().map(|package| package.file) + } } #[derive(Debug)] @@ -31,14 +37,6 @@ pub struct PinnedPackage { } impl PinnedPackage { - pub fn filename(&self) -> &str { - &self.file.filename - } - - pub fn url(&self) -> &str { - &self.file.url - } - pub fn version(&self) -> &Version { &self.metadata.version } diff --git a/requirements.in b/requirements.in index 7e66a17d4..79d6fb80b 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,5 @@ -black +packaging>=23.1 +pygls>=1.0.1 +lsprotocol>=2023.0.0a1 +ruff>=0.0.274 +typing_extensions