Move puffin-installer to its own crate (#23)

This commit is contained in:
Charlie Marsh 2023-10-06 15:31:21 -04:00 committed by GitHub
parent f395c9c98c
commit ff8e24a621
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 105 additions and 63 deletions

View file

@ -8,6 +8,10 @@ Command-line interface for the Puffin package manager.
Client for interacting with PyPI-compatible HTTP APIs. Client for interacting with PyPI-compatible HTTP APIs.
## [puffin-installer](./puffin-installer)
Functionality for installing Python packages into a virtual environment.
## [puffin-interpreter](./puffin-interpreter) ## [puffin-interpreter](./puffin-interpreter)
Functionality for detecting and leveraging the current Python interpreter. Functionality for detecting and leveraging the current Python interpreter.
@ -20,6 +24,6 @@ Types and functionality for working with Python packages, e.g., parsing wheel fi
Functionality for detecting the current platform (operating system, architecture, etc.). Functionality for detecting the current platform (operating system, architecture, etc.).
## [puffin-resolve](./puffin-resolve) ## [puffin-resolver](./puffin-resolver)
Functionality for resolving Python packages and their dependencies. Functionality for resolving Python packages and their dependencies.

View file

@ -5,10 +5,11 @@ edition = "2021"
[dependencies] [dependencies]
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-installer = { path = "../puffin-installer" }
puffin-interpreter = { path = "../puffin-interpreter" } puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" } puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" } puffin-package = { path = "../puffin-package" }
puffin-resolve = { path = "../puffin-resolve" } puffin-resolver = { path = "../puffin-resolver" }
anyhow = { version = "1.0.75" } anyhow = { version = "1.0.75" }
async-std = { version = "1.12.0", features = [ async-std = { version = "1.12.0", features = [
@ -26,6 +27,5 @@ pep440_rs = { version = "0.3.12" }
tracing = { version = "0.1.37" } tracing = { version = "0.1.37" }
tracing-tree = { version = "0.2.5" } tracing-tree = { version = "0.2.5" }
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
rayon = { version = "1.8.0" }
url = { version = "2.4.1" } url = { version = "2.4.1" }
tempfile = { version = "3.8.0" } tempfile = { version = "3.8.0" }

View file

@ -8,7 +8,6 @@ use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags; use puffin_platform::tags::Tags;
use puffin_platform::Platform; use puffin_platform::Platform;
use puffin_resolve::resolve;
use crate::commands::ExitStatus; use crate::commands::ExitStatus;
@ -44,7 +43,7 @@ pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result<ExitStat
}; };
// Resolve the dependencies. // Resolve the dependencies.
let resolution = resolve(&requirements, markers, &tags, &client).await?; let resolution = puffin_resolver::resolve(&requirements, markers, &tags, &client).await?;
for (name, package) in resolution.iter() { for (name, package) in resolution.iter() {
#[allow(clippy::print_stdout)] #[allow(clippy::print_stdout)]

View file

@ -2,16 +2,12 @@ use std::path::Path;
use std::str::FromStr; use std::str::FromStr;
use anyhow::Result; use anyhow::Result;
use async_std::fs::File;
use install_wheel_rs::{install_wheel, InstallLocation};
use tracing::debug; use tracing::debug;
use url::Url;
use puffin_client::PypiClientBuilder; use puffin_client::PypiClientBuilder;
use puffin_interpreter::PythonExecutable; use puffin_interpreter::PythonExecutable;
use puffin_platform::tags::Tags; use puffin_platform::tags::Tags;
use puffin_platform::Platform; use puffin_platform::Platform;
use puffin_resolve::resolve;
use crate::commands::ExitStatus; use crate::commands::ExitStatus;
@ -48,45 +44,11 @@ pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result<ExitStat
// Resolve the dependencies. // Resolve the dependencies.
// TODO(charlie): When installing, assume `--no-deps`. // TODO(charlie): When installing, assume `--no-deps`.
let resolution = resolve(&requirements, markers, &tags, &client).await?; let resolution = puffin_resolver::resolve(&requirements, markers, &tags, &client).await?;
// Create a temporary directory, in which we'll store the wheels. // Install into the current environment.
let tmp_dir = tempfile::tempdir()?; let wheels = resolution.into_files().collect::<Vec<_>>();
puffin_installer::install(&wheels, &python, &client).await?;
// Download each wheel.
// TODO(charlie): Store these in a content-addressed cache.
// TODO(charlie): Use channels to efficiently stream-and-install.
for (name, package) in resolution.iter() {
let url = Url::parse(package.url())?;
let reader = client.stream_external(&url).await?;
// TODO(charlie): Stream the unzip.
let mut writer = File::create(tmp_dir.path().join(format!("{name}.whl"))).await?;
async_std::io::copy(reader, &mut writer).await?;
}
// Install each wheel.
// TODO(charlie): Use channels to efficiently stream-and-install.
let location = InstallLocation::Venv {
venv_base: python.venv().to_path_buf(),
python_version: python.simple_version(),
};
let locked_dir = location.acquire_lock()?;
for (name, package) in resolution.iter() {
let path = tmp_dir.path().join(format!("{name}.whl"));
let filename = install_wheel_rs::WheelFilename::from_str(package.filename())?;
// TODO(charlie): Should this be async?
install_wheel(
&locked_dir,
std::fs::File::open(path)?,
filename,
false,
&[],
"",
python.executable(),
)?;
}
Ok(ExitStatus::Success) Ok(ExitStatus::Success)
} }

View file

@ -0,0 +1,25 @@
[package]
name = "puffin-installer"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" }
anyhow = { version = "1.0.75" }
async-std = { version = "1.12.0", features = [
"attributes",
"tokio1",
"unstable",
] }
install-wheel-rs = { version = "0.0.1" }
tempfile = { version = "3.8.0" }
tracing = { version = "0.1.37" }
url = { version = "2.4.1" }

View file

@ -0,0 +1,56 @@
use std::str::FromStr;
use anyhow::Result;
use install_wheel_rs::{install_wheel, InstallLocation};
use url::Url;
use puffin_client::{File, PypiClient};
use puffin_interpreter::PythonExecutable;
/// Install a set of wheels into a Python virtual environment.
pub async fn install(
wheels: &[File],
python: &PythonExecutable,
client: &PypiClient,
) -> Result<()> {
// Create a temporary directory, in which we'll store the wheels.
let tmp_dir = tempfile::tempdir()?;
// Download each wheel.
// TODO(charlie): Store these in a content-addressed cache.
// TODO(charlie): Use channels to efficiently stream-and-install.
for wheel in wheels {
let url = Url::parse(&wheel.url)?;
let reader = client.stream_external(&url).await?;
// TODO(charlie): Stream the unzip.
let mut writer =
async_std::fs::File::create(tmp_dir.path().join(&wheel.hashes.sha256)).await?;
async_std::io::copy(reader, &mut writer).await?;
}
// Install each wheel.
// TODO(charlie): Use channels to efficiently stream-and-install.
let location = InstallLocation::Venv {
venv_base: python.venv().to_path_buf(),
python_version: python.simple_version(),
};
let locked_dir = location.acquire_lock()?;
for wheel in wheels {
let path = tmp_dir.path().join(&wheel.hashes.sha256);
let filename = install_wheel_rs::WheelFilename::from_str(&wheel.filename)?;
// TODO(charlie): Should this be async?
install_wheel(
&locked_dir,
std::fs::File::open(path)?,
filename,
false,
&[],
"",
python.executable(),
)?;
}
Ok(())
}

View file

@ -1,5 +1,5 @@
[package] [package]
name = "puffin-resolve" name = "puffin-resolver"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
rust-version.workspace = true rust-version.workspace = true
@ -11,15 +11,9 @@ license.workspace = true
[dependencies] [dependencies]
puffin-client = { path = "../puffin-client" } puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" } puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" } puffin-package = { path = "../puffin-package" }
async-std = { version = "1.12.0", features = [
"attributes",
"tokio1",
"unstable",
] }
pep440_rs = "0.3.12" pep440_rs = "0.3.12"
futures = "0.3.28" futures = "0.3.28"
anyhow = "1.0.75" anyhow = "1.0.75"

View file

@ -19,9 +19,15 @@ use puffin_platform::tags::Tags;
pub struct Resolution(HashMap<PackageName, PinnedPackage>); pub struct Resolution(HashMap<PackageName, PinnedPackage>);
impl Resolution { impl Resolution {
/// Iterate over the pinned packages in this resolution.
pub fn iter(&self) -> impl Iterator<Item = (&PackageName, &PinnedPackage)> { pub fn iter(&self) -> impl Iterator<Item = (&PackageName, &PinnedPackage)> {
self.0.iter() self.0.iter()
} }
/// Iterate over the wheels in this resolution.
pub fn into_files(self) -> impl Iterator<Item = File> {
self.0.into_values().map(|package| package.file)
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -31,14 +37,6 @@ pub struct PinnedPackage {
} }
impl PinnedPackage { impl PinnedPackage {
pub fn filename(&self) -> &str {
&self.file.filename
}
pub fn url(&self) -> &str {
&self.file.url
}
pub fn version(&self) -> &Version { pub fn version(&self) -> &Version {
&self.metadata.version &self.metadata.version
} }

View file

@ -1 +1,5 @@
black packaging>=23.1
pygls>=1.0.1
lsprotocol>=2023.0.0a1
ruff>=0.0.274
typing_extensions