From 9ea6eaeb105042dc682dd1029dfb2733c7659bf8 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 5 Oct 2023 21:44:31 -0400 Subject: [PATCH] Add separate compile and install commands (#17) Closes #9. --- crates/README.md | 4 + crates/puffin-cli/Cargo.toml | 2 +- crates/puffin-cli/src/commands/compile.rs | 46 ++++++ crates/puffin-cli/src/commands/install.rs | 159 ++------------------ crates/puffin-cli/src/commands/mod.rs | 2 + crates/puffin-cli/src/main.rs | 33 +++- crates/puffin-interpreter/Cargo.toml | 1 + crates/puffin-interpreter/src/lib.rs | 12 +- crates/puffin-package/benches/parser.rs | 2 +- crates/puffin-platform/Cargo.toml | 1 + crates/puffin-platform/src/lib.rs | 7 +- crates/puffin-resolve/Cargo.toml | 27 ++++ crates/puffin-resolve/src/lib.rs | 175 ++++++++++++++++++++++ 13 files changed, 304 insertions(+), 167 deletions(-) create mode 100644 crates/puffin-cli/src/commands/compile.rs create mode 100644 crates/puffin-resolve/Cargo.toml create mode 100644 crates/puffin-resolve/src/lib.rs diff --git a/crates/README.md b/crates/README.md index db2ce6be6..4b40f78e8 100644 --- a/crates/README.md +++ b/crates/README.md @@ -19,3 +19,7 @@ Types and functionality for working with Python packages, e.g., parsing wheel fi ## [puffin-platform](./puffin-platform) Functionality for detecting the current platform (operating system, architecture, etc.). + +## [puffin-resolve](./puffin-resolve) + +Functionality for resolving Python packages and their dependencies. diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index 0c944c793..5016ab16f 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -8,11 +8,11 @@ puffin-client = { path = "../puffin-client" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-platform = { path = "../puffin-platform" } puffin-package = { path = "../puffin-package" } +puffin-resolve = { path = "../puffin-resolve" } anyhow = { version = "1.0.75" } clap = { version = "4.4.6", features = ["derive"] } colored = { version = "2.0.4" } -memchr = { version = "2.6.4" } async-std = { version = "1.12.0", features = [ "attributes", "tokio1", diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs new file mode 100644 index 000000000..80bf90214 --- /dev/null +++ b/crates/puffin-cli/src/commands/compile.rs @@ -0,0 +1,46 @@ +use std::path::Path; +use std::str::FromStr; + +use anyhow::Result; +use tracing::debug; + +use puffin_interpreter::PythonExecutable; +use puffin_platform::Platform; +use puffin_resolve::resolve; + +use crate::commands::ExitStatus; + +pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result { + // Read the `requirements.txt` from disk. + let requirements_txt = std::fs::read_to_string(src)?; + + // Parse the `requirements.txt` into a list of requirements. + let requirements = puffin_package::requirements::Requirements::from_str(&requirements_txt)?; + + // Detect the current Python interpreter. + let platform = Platform::current()?; + let python = PythonExecutable::from_env(&platform)?; + debug!( + "Using Python interpreter: {}", + python.executable().display() + ); + + // Resolve the dependencies. + let resolution = resolve( + &requirements, + python.version(), + python.markers(), + &platform, + cache, + ) + .await?; + + for (name, version) in resolution.iter() { + #[allow(clippy::print_stdout)] + { + println!("{name}=={version}"); + } + } + + Ok(ExitStatus::Success) +} diff --git a/crates/puffin-cli/src/commands/install.rs b/crates/puffin-cli/src/commands/install.rs index 0431459c7..ca1635934 100644 --- a/crates/puffin-cli/src/commands/install.rs +++ b/crates/puffin-cli/src/commands/install.rs @@ -1,35 +1,15 @@ -use std::collections::{HashMap, HashSet}; use std::path::Path; use std::str::FromStr; use anyhow::Result; -use futures::future::Either; -use futures::{StreamExt, TryFutureExt}; -use pep440_rs::Version; -use pep508_rs::{Requirement, VersionOrUrl}; use tracing::debug; -use puffin_client::{File, PypiClientBuilder, SimpleJson}; use puffin_interpreter::PythonExecutable; -use puffin_package::metadata::Metadata21; -use puffin_package::package_name::PackageName; -use puffin_package::wheel::WheelFilename; use puffin_platform::Platform; +use puffin_resolve::resolve; use crate::commands::ExitStatus; -#[derive(Debug)] -enum Request { - Package(Requirement), - Version(Requirement, File), -} - -#[derive(Debug)] -enum Response { - Package(SimpleJson, Requirement), - Version(Metadata21, Requirement), -} - pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result { // Read the `requirements.txt` from disk. let requirements_txt = std::fs::read_to_string(src)?; @@ -45,134 +25,17 @@ pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result Either::Left( - pypi_client - .simple(requirement.name.clone()) - .map_ok(move |metadata| Response::Package(metadata, requirement)), - ), - Request::Version(requirement, file) => Either::Right( - pypi_client - .file(file) - .map_ok(move |metadata| Response::Version(metadata, requirement)), - ), - }) - .buffer_unordered(32) - .ready_chunks(32); - - // Push all the requirements into the package sink. - let mut in_flight: HashSet = HashSet::with_capacity(requirements.len()); - for requirement in &*requirements { - debug!("--> adding root dependency: {}", requirement); - package_sink.unbounded_send(Request::Package(requirement.clone()))?; - in_flight.insert(PackageName::normalize(&requirement.name)); - } - - // Resolve the requirements. - let mut resolution: HashMap = HashMap::with_capacity(requirements.len()); - - while let Some(chunk) = package_stream.next().await { - for result in chunk { - let result: Response = result?; - match result { - Response::Package(metadata, requirement) => { - // TODO(charlie): Support URLs. Right now, we treat a URL as an unpinned dependency. - let specifiers = - requirement - .version_or_url - .as_ref() - .and_then(|version_or_url| match version_or_url { - VersionOrUrl::VersionSpecifier(specifiers) => Some(specifiers), - VersionOrUrl::Url(_) => None, - }); - - // Pick a version that satisfies the requirement. - let Some(file) = metadata.files.iter().rev().find(|file| { - // We only support wheels for now. - let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else { - return false; - }; - - let Ok(version) = Version::from_str(&name.version) else { - return false; - }; - - if !name.is_compatible(&tags) { - return false; - } - - specifiers - .iter() - .all(|specifier| specifier.contains(&version)) - }) else { - continue; - }; - - package_sink.unbounded_send(Request::Version(requirement, file.clone()))?; - } - Response::Version(metadata, requirement) => { - debug!( - "--> selected version {} for {}", - metadata.version, requirement - ); - - // Add to the resolved set. - let normalized_name = PackageName::normalize(&requirement.name); - in_flight.remove(&normalized_name); - resolution.insert(normalized_name, metadata.version); - - // Enqueue its dependencies. - for dependency in metadata.requires_dist { - if !dependency.evaluate_markers( - python.markers(), - requirement.extras.clone().unwrap_or_default(), - ) { - debug!("--> ignoring {dependency} due to environment mismatch"); - continue; - } - - let normalized_name = PackageName::normalize(&dependency.name); - - if resolution.contains_key(&normalized_name) { - continue; - } - - if !in_flight.insert(normalized_name) { - continue; - } - - debug!("--> adding transitive dependency: {}", dependency); - - package_sink.unbounded_send(Request::Package(dependency))?; - } - } - } - } - - if in_flight.is_empty() { - break; - } - } - - for (name, version) in resolution { + for (name, version) in resolution.iter() { #[allow(clippy::print_stdout)] { println!("{name}=={version}"); diff --git a/crates/puffin-cli/src/commands/mod.rs b/crates/puffin-cli/src/commands/mod.rs index d98ef5fe2..2d5a43e5b 100644 --- a/crates/puffin-cli/src/commands/mod.rs +++ b/crates/puffin-cli/src/commands/mod.rs @@ -1,7 +1,9 @@ use std::process::ExitCode; +pub(crate) use compile::compile; pub(crate) use install::install; +mod compile; mod install; #[derive(Copy, Clone)] diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs index 71c24abb4..73610e17f 100644 --- a/crates/puffin-cli/src/main.rs +++ b/crates/puffin-cli/src/main.rs @@ -20,13 +20,25 @@ struct Cli { #[derive(Subcommand)] enum Commands { - /// Install dependencies from a `requirements.text` file. + /// Compile a `requirements.in` file to a `requirements.txt` file. + Compile(CompileArgs), + /// Install dependencies from a `requirements.txt` file. Install(InstallArgs), } +#[derive(Args)] +struct CompileArgs { + /// Path to the `requirements.txt` file to compile. + src: PathBuf, + + /// Avoid reading from or writing to the cache. + #[arg(long)] + no_cache: bool, +} + #[derive(Args)] struct InstallArgs { - /// Path to the `requirements.text` file to install. + /// Path to the `requirements.txt` file to install. src: PathBuf, /// Avoid reading from or writing to the cache. @@ -43,12 +55,21 @@ async fn main() -> ExitCode { let dirs = ProjectDirs::from("", "", "puffin"); let result = match &cli.command { - Commands::Install(install) => { - commands::install( - &install.src, + Commands::Compile(args) => { + commands::compile( + &args.src, dirs.as_ref() .map(directories::ProjectDirs::cache_dir) - .filter(|_| !install.no_cache), + .filter(|_| !args.no_cache), + ) + .await + } + Commands::Install(args) => { + commands::install( + &args.src, + dirs.as_ref() + .map(directories::ProjectDirs::cache_dir) + .filter(|_| !args.no_cache), ) .await } diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index d3110b878..2e9fb9dbc 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -16,3 +16,4 @@ anyhow = { version = "1.0.75" } pep508_rs = { version = "0.2.3", features = ["serde"] } serde_json = { version = "1.0.107" } tracing = { version = "0.1.37" } +pep440_rs = "0.3.12" diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index f89a54ea4..7257f0f92 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; +use pep440_rs::Version; use pep508_rs::MarkerEnvironment; use puffin_platform::Platform; @@ -42,13 +43,8 @@ impl PythonExecutable { &self.markers } - /// Returns the Python version as a tuple of (major, minor). - pub fn version(&self) -> (u8, u8) { - // TODO(charlie): Use `Version`. - let python_version = &self.markers.python_version; - ( - u8::try_from(python_version.release[0]).expect("Python major version is too large"), - u8::try_from(python_version.release[1]).expect("Python minor version is too large"), - ) + /// Returns the Python version. + pub fn version(&self) -> &Version { + &self.markers.python_version.version } } diff --git a/crates/puffin-package/benches/parser.rs b/crates/puffin-package/benches/parser.rs index 4b9349024..fcecec6e0 100644 --- a/crates/puffin-package/benches/parser.rs +++ b/crates/puffin-package/benches/parser.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use puffin_package::Requirements; +use puffin_package::requirements::Requirements; const REQUIREMENTS_TXT: &str = r" # diff --git a/crates/puffin-platform/Cargo.toml b/crates/puffin-platform/Cargo.toml index bda775869..35491817b 100644 --- a/crates/puffin-platform/Cargo.toml +++ b/crates/puffin-platform/Cargo.toml @@ -14,6 +14,7 @@ license.workspace = true [dependencies] glibc_version = "0.1.2" goblin = "0.6.0" +pep440_rs = "0.3.12" platform-info = "2.0.2" plist = "1.5.0" regex = "1.9.6" diff --git a/crates/puffin-platform/src/lib.rs b/crates/puffin-platform/src/lib.rs index bd95a3a04..fe48fcfe2 100644 --- a/crates/puffin-platform/src/lib.rs +++ b/crates/puffin-platform/src/lib.rs @@ -38,7 +38,7 @@ impl Platform { pub fn compatible_tags( &self, - python_version: (u8, u8), + python_version: &pep440_rs::Version, ) -> Result, PlatformError> { compatible_tags(python_version, &self.os, self.arch) } @@ -476,11 +476,12 @@ pub fn compatible_platform_tags(os: &Os, arch: Arch) -> Result, Plat /// Returns the compatible tags in a (`python_tag`, `abi_tag`, `platform_tag`) format pub fn compatible_tags( - python_version: (u8, u8), + python_version: &pep440_rs::Version, os: &Os, arch: Arch, ) -> Result, PlatformError> { - assert_eq!(python_version.0, 3); + let python_version = (python_version.release[0], python_version.release[1]); + let mut tags = Vec::new(); let platform_tags = compatible_platform_tags(os, arch)?; // 1. This exact c api version diff --git a/crates/puffin-resolve/Cargo.toml b/crates/puffin-resolve/Cargo.toml new file mode 100644 index 000000000..e518112b3 --- /dev/null +++ b/crates/puffin-resolve/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "puffin-resolve" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +puffin-client = { path = "../puffin-client" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-platform = { path = "../puffin-platform" } +puffin-package = { path = "../puffin-package" } + +async-std = { version = "1.12.0", features = [ + "attributes", + "tokio1", + "unstable", +] } +pep440_rs = "0.3.12" +futures = "0.3.28" +anyhow = "1.0.75" +tracing = "0.1.37" +pep508_rs = "0.2.3" diff --git a/crates/puffin-resolve/src/lib.rs b/crates/puffin-resolve/src/lib.rs new file mode 100644 index 000000000..ed525c200 --- /dev/null +++ b/crates/puffin-resolve/src/lib.rs @@ -0,0 +1,175 @@ +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::str::FromStr; + +use anyhow::Result; +use futures::future::Either; +use futures::{StreamExt, TryFutureExt}; +use pep440_rs::Version; +use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl}; +use tracing::debug; + +use puffin_client::{File, PypiClientBuilder, SimpleJson}; +use puffin_package::metadata::Metadata21; +use puffin_package::package_name::PackageName; +use puffin_package::requirements::Requirements; +use puffin_package::wheel::WheelFilename; +use puffin_platform::Platform; + +pub struct Resolution(HashMap); + +impl Resolution { + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +/// Resolve a set of requirements into a set of pinned versions. +pub async fn resolve( + requirements: &Requirements, + python_version: &Version, + markers: &MarkerEnvironment, + platform: &Platform, + cache: Option<&Path>, +) -> Result { + // Instantiate a client. + let pypi_client = { + let mut pypi_client = PypiClientBuilder::default(); + if let Some(cache) = cache { + pypi_client = pypi_client.cache(cache); + } + pypi_client.build() + }; + + // A channel to fetch package metadata (e.g., given `flask`, fetch all versions) and version + // metadata (e.g., given `flask==1.0.0`, fetch the metadata for that version). + let (package_sink, package_stream) = futures::channel::mpsc::unbounded(); + + // Initialize the package stream. + let mut package_stream = package_stream + .map(|request: Request| match request { + Request::Package(requirement) => Either::Left( + pypi_client + .simple(requirement.name.clone()) + .map_ok(move |metadata| Response::Package(metadata, requirement)), + ), + Request::Version(requirement, file) => Either::Right( + pypi_client + .file(file) + .map_ok(move |metadata| Response::Version(metadata, requirement)), + ), + }) + .buffer_unordered(32) + .ready_chunks(32); + + // Push all the requirements into the package sink. + let mut in_flight: HashSet = HashSet::with_capacity(requirements.len()); + for requirement in requirements.iter() { + debug!("--> adding root dependency: {}", requirement); + package_sink.unbounded_send(Request::Package(requirement.clone()))?; + in_flight.insert(PackageName::normalize(&requirement.name)); + } + + // Determine the compatible platform tags. + let tags = platform.compatible_tags(python_version)?; + + // Resolve the requirements. + let mut resolution: HashMap = HashMap::with_capacity(requirements.len()); + + while let Some(chunk) = package_stream.next().await { + for result in chunk { + let result: Response = result?; + match result { + Response::Package(metadata, requirement) => { + // TODO(charlie): Support URLs. Right now, we treat a URL as an unpinned dependency. + let specifiers = + requirement + .version_or_url + .as_ref() + .and_then(|version_or_url| match version_or_url { + VersionOrUrl::VersionSpecifier(specifiers) => Some(specifiers), + VersionOrUrl::Url(_) => None, + }); + + // Pick a version that satisfies the requirement. + let Some(file) = metadata.files.iter().rev().find(|file| { + // We only support wheels for now. + let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else { + return false; + }; + + let Ok(version) = Version::from_str(&name.version) else { + return false; + }; + + if !name.is_compatible(&tags) { + return false; + } + + specifiers + .iter() + .all(|specifier| specifier.contains(&version)) + }) else { + continue; + }; + + package_sink.unbounded_send(Request::Version(requirement, file.clone()))?; + } + Response::Version(metadata, requirement) => { + debug!( + "--> selected version {} for {}", + metadata.version, requirement + ); + + // Add to the resolved set. + let normalized_name = PackageName::normalize(&requirement.name); + in_flight.remove(&normalized_name); + resolution.insert(normalized_name, metadata.version); + + // Enqueue its dependencies. + for dependency in metadata.requires_dist { + if !dependency.evaluate_markers( + markers, + requirement.extras.clone().unwrap_or_default(), + ) { + debug!("--> ignoring {dependency} due to environment mismatch"); + continue; + } + + let normalized_name = PackageName::normalize(&dependency.name); + + if resolution.contains_key(&normalized_name) { + continue; + } + + if !in_flight.insert(normalized_name) { + continue; + } + + debug!("--> adding transitive dependency: {}", dependency); + + package_sink.unbounded_send(Request::Package(dependency))?; + } + } + } + } + + if in_flight.is_empty() { + break; + } + } + + Ok(Resolution(resolution)) +} + +#[derive(Debug)] +enum Request { + Package(Requirement), + Version(Requirement, File), +} + +#[derive(Debug)] +enum Response { + Package(SimpleJson, Requirement), + Version(Metadata21, Requirement), +}