mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Add separate compile and install commands (#17)

Browse source 
Closes #9.
This commit is contained in:
Charlie Marsh 2023-10-05 21:44:31 -04:00 • committed by GitHub
parent 4c30cb146a
commit 9ea6eaeb10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 304 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/README.md
View file

@ -19,3 +19,7 @@ Types and functionality for working with Python packages, e.g., parsing wheel fi
## [puffin-platform](./puffin-platform)
Functionality for detecting the current platform (operating system, architecture, etc.).
## [puffin-resolve](./puffin-resolve)
Functionality for resolving Python packages and their dependencies.

2
crates/puffin-cli/Cargo.toml
View file

@ -8,11 +8,11 @@ puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" }
puffin-resolve = { path = "../puffin-resolve" }
anyhow = { version = "1.0.75" }
clap = { version = "4.4.6", features = ["derive"] }
colored = { version = "2.0.4" }
memchr = { version = "2.6.4" }
async-std = { version = "1.12.0", features = [
"attributes",
"tokio1",

46
crates/puffin-cli/src/commands/compile.rs Normal file
View file

@ -0,0 +1,46 @@
use std::path::Path;
use std::str::FromStr;
use anyhow::Result;
use tracing::debug;
use puffin_interpreter::PythonExecutable;
use puffin_platform::Platform;
use puffin_resolve::resolve;
use crate::commands::ExitStatus;
pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result<ExitStatus> {
// Read the `requirements.txt` from disk.
let requirements_txt = std::fs::read_to_string(src)?;
// Parse the `requirements.txt` into a list of requirements.
let requirements = puffin_package::requirements::Requirements::from_str(&requirements_txt)?;
// Detect the current Python interpreter.
let platform = Platform::current()?;
let python = PythonExecutable::from_env(&platform)?;
debug!(
"Using Python interpreter: {}",
python.executable().display()
);
// Resolve the dependencies.
let resolution = resolve(
&requirements,
python.version(),
python.markers(),
&platform,
cache,
)
.await?;
for (name, version) in resolution.iter() {
#[allow(clippy::print_stdout)]
{
println!("{name}=={version}");
}
}
Ok(ExitStatus::Success)
}

157
crates/puffin-cli/src/commands/install.rs
View file

@ -1,35 +1,15 @@
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::str::FromStr;
use anyhow::Result;
use futures::future::Either;
use futures::{StreamExt, TryFutureExt};
use pep440_rs::Version;
use pep508_rs::{Requirement, VersionOrUrl};
use tracing::debug;
use puffin_client::{File, PypiClientBuilder, SimpleJson};
use puffin_interpreter::PythonExecutable;
use puffin_package::metadata::Metadata21;
use puffin_package::package_name::PackageName;
use puffin_package::wheel::WheelFilename;
use puffin_platform::Platform;
use puffin_resolve::resolve;
use crate::commands::ExitStatus;
#[derive(Debug)]
enum Request {
Package(Requirement),
Version(Requirement, File),
}
#[derive(Debug)]
enum Response {
Package(SimpleJson, Requirement),
Version(Metadata21, Requirement),
}
pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result<ExitStatus> {
// Read the `requirements.txt` from disk.
let requirements_txt = std::fs::read_to_string(src)?;
@ -45,134 +25,17 @@ pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result<ExitStat
python.executable().display()
);
// Determine the compatible platform tags.
let tags = platform.compatible_tags(python.version())?;
// Instantiate a client.
let pypi_client = {
let mut pypi_client = PypiClientBuilder::default();
if let Some(cache) = cache {
pypi_client = pypi_client.cache(cache);
}
pypi_client.build()
};
// A channel to fetch package metadata (e.g., given `flask`, fetch all versions) and version
// metadata (e.g., given `flask==1.0.0`, fetch the metadata for that version).
let (package_sink, package_stream) = futures::channel::mpsc::unbounded();
// Initialize the package stream.
let mut package_stream = package_stream
.map(|request: Request| match request {
Request::Package(requirement) => Either::Left(
pypi_client
.simple(requirement.name.clone())
.map_ok(move |metadata| Response::Package(metadata, requirement)),
),
Request::Version(requirement, file) => Either::Right(
pypi_client
.file(file)
.map_ok(move |metadata| Response::Version(metadata, requirement)),
),
})
.buffer_unordered(32)
.ready_chunks(32);
// Push all the requirements into the package sink.
let mut in_flight: HashSet<PackageName> = HashSet::with_capacity(requirements.len());
for requirement in &*requirements {
debug!("--> adding root dependency: {}", requirement);
package_sink.unbounded_send(Request::Package(requirement.clone()))?;
in_flight.insert(PackageName::normalize(&requirement.name));
}
// Resolve the requirements.
let mut resolution: HashMap<PackageName, Version> = HashMap::with_capacity(requirements.len());
while let Some(chunk) = package_stream.next().await {
for result in chunk {
let result: Response = result?;
match result {
Response::Package(metadata, requirement) => {
// TODO(charlie): Support URLs. Right now, we treat a URL as an unpinned dependency.
let specifiers =
requirement
.version_or_url
.as_ref()
.and_then(|version_or_url| match version_or_url {
VersionOrUrl::VersionSpecifier(specifiers) => Some(specifiers),
VersionOrUrl::Url(_) => None,
});
// Pick a version that satisfies the requirement.
let Some(file) = metadata.files.iter().rev().find(|file| {
// We only support wheels for now.
let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else {
return false;
};
let Ok(version) = Version::from_str(&name.version) else {
return false;
};
if !name.is_compatible(&tags) {
return false;
}
specifiers
.iter()
.all(|specifier| specifier.contains(&version))
}) else {
continue;
};
package_sink.unbounded_send(Request::Version(requirement, file.clone()))?;
}
Response::Version(metadata, requirement) => {
debug!(
"--> selected version {} for {}",
metadata.version, requirement
);
// Add to the resolved set.
let normalized_name = PackageName::normalize(&requirement.name);
in_flight.remove(&normalized_name);
resolution.insert(normalized_name, metadata.version);
// Enqueue its dependencies.
for dependency in metadata.requires_dist {
if !dependency.evaluate_markers(
// Resolve the dependencies.
let resolution = resolve(
&requirements,
python.version(),
python.markers(),
requirement.extras.clone().unwrap_or_default(),
) {
debug!("--> ignoring {dependency} due to environment mismatch");
continue;
}
&platform,
cache,
)
.await?;
let normalized_name = PackageName::normalize(&dependency.name);
if resolution.contains_key(&normalized_name) {
continue;
}
if !in_flight.insert(normalized_name) {
continue;
}
debug!("--> adding transitive dependency: {}", dependency);
package_sink.unbounded_send(Request::Package(dependency))?;
}
}
}
}
if in_flight.is_empty() {
break;
}
}
for (name, version) in resolution {
for (name, version) in resolution.iter() {
#[allow(clippy::print_stdout)]
{
println!("{name}=={version}");

2
crates/puffin-cli/src/commands/mod.rs
View file

@ -1,7 +1,9 @@
use std::process::ExitCode;
pub(crate) use compile::compile;
pub(crate) use install::install;
mod compile;
mod install;
#[derive(Copy, Clone)]

33
crates/puffin-cli/src/main.rs
View file

@ -20,13 +20,25 @@ struct Cli {
#[derive(Subcommand)]
enum Commands {
/// Install dependencies from a `requirements.text` file.
/// Compile a `requirements.in` file to a `requirements.txt` file.
Compile(CompileArgs),
/// Install dependencies from a `requirements.txt` file.
Install(InstallArgs),
}
#[derive(Args)]
struct CompileArgs {
/// Path to the `requirements.txt` file to compile.
src: PathBuf,
/// Avoid reading from or writing to the cache.
#[arg(long)]
no_cache: bool,
}
#[derive(Args)]
struct InstallArgs {
/// Path to the `requirements.text` file to install.
/// Path to the `requirements.txt` file to install.
src: PathBuf,
/// Avoid reading from or writing to the cache.
@ -43,12 +55,21 @@ async fn main() -> ExitCode {
let dirs = ProjectDirs::from("", "", "puffin");
let result = match &cli.command {
Commands::Install(install) => {
commands::install(
&install.src,
Commands::Compile(args) => {
commands::compile(
&args.src,
dirs.as_ref()
.map(directories::ProjectDirs::cache_dir)
.filter(|_| !install.no_cache),
.filter(|_| !args.no_cache),
)
.await
}
Commands::Install(args) => {
commands::install(
&args.src,
dirs.as_ref()
.map(directories::ProjectDirs::cache_dir)
.filter(|_| !args.no_cache),
)
.await
}

1
crates/puffin-interpreter/Cargo.toml
View file

@ -16,3 +16,4 @@ anyhow = { version = "1.0.75" }
pep508_rs = { version = "0.2.3", features = ["serde"] }
serde_json = { version = "1.0.107" }
tracing = { version = "0.1.37" }
pep440_rs = "0.3.12"

12
crates/puffin-interpreter/src/lib.rs
View file

@ -1,6 +1,7 @@
use std::path::{Path, PathBuf};
use anyhow::Result;
use pep440_rs::Version;
use pep508_rs::MarkerEnvironment;
use puffin_platform::Platform;
@ -42,13 +43,8 @@ impl PythonExecutable {
&self.markers
}
/// Returns the Python version as a tuple of (major, minor).
pub fn version(&self) -> (u8, u8) {
// TODO(charlie): Use `Version`.
let python_version = &self.markers.python_version;
(
u8::try_from(python_version.release[0]).expect("Python major version is too large"),
u8::try_from(python_version.release[1]).expect("Python minor version is too large"),
)
/// Returns the Python version.
pub fn version(&self) -> &Version {
&self.markers.python_version.version
}
}

2
crates/puffin-package/benches/parser.rs
View file

@ -2,7 +2,7 @@ use std::str::FromStr;
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use puffin_package::Requirements;
use puffin_package::requirements::Requirements;
const REQUIREMENTS_TXT: &str = r"
#

1
crates/puffin-platform/Cargo.toml
View file

@ -14,6 +14,7 @@ license.workspace = true
[dependencies]
glibc_version = "0.1.2"
goblin = "0.6.0"
pep440_rs = "0.3.12"
platform-info = "2.0.2"
plist = "1.5.0"
regex = "1.9.6"

7
crates/puffin-platform/src/lib.rs
View file

@ -38,7 +38,7 @@ impl Platform {
pub fn compatible_tags(
&self,
python_version: (u8, u8),
python_version: &pep440_rs::Version,
) -> Result<Vec<(String, String, String)>, PlatformError> {
compatible_tags(python_version, &self.os, self.arch)
}
@ -476,11 +476,12 @@ pub fn compatible_platform_tags(os: &Os, arch: Arch) -> Result<Vec<String>, Plat
/// Returns the compatible tags in a (`python_tag`, `abi_tag`, `platform_tag`) format
pub fn compatible_tags(
python_version: (u8, u8),
python_version: &pep440_rs::Version,
os: &Os,
arch: Arch,
) -> Result<Vec<(String, String, String)>, PlatformError> {
assert_eq!(python_version.0, 3);
let python_version = (python_version.release[0], python_version.release[1]);
let mut tags = Vec::new();
let platform_tags = compatible_platform_tags(os, arch)?;
// 1. This exact c api version

27
crates/puffin-resolve/Cargo.toml Normal file
View file

@ -0,0 +1,27 @@
[package]
name = "puffin-resolve"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
homepage.workspace = true
documentation.workspace = true
repository.workspace = true
authors.workspace = true
license.workspace = true
[dependencies]
puffin-client = { path = "../puffin-client" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-platform = { path = "../puffin-platform" }
puffin-package = { path = "../puffin-package" }
async-std = { version = "1.12.0", features = [
"attributes",
"tokio1",
"unstable",
] }
pep440_rs = "0.3.12"
futures = "0.3.28"
anyhow = "1.0.75"
tracing = "0.1.37"
pep508_rs = "0.2.3"

175
crates/puffin-resolve/src/lib.rs Normal file
View file

@ -0,0 +1,175 @@
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::str::FromStr;
use anyhow::Result;
use futures::future::Either;
use futures::{StreamExt, TryFutureExt};
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use tracing::debug;
use puffin_client::{File, PypiClientBuilder, SimpleJson};
use puffin_package::metadata::Metadata21;
use puffin_package::package_name::PackageName;
use puffin_package::requirements::Requirements;
use puffin_package::wheel::WheelFilename;
use puffin_platform::Platform;
pub struct Resolution(HashMap<PackageName, Version>);
impl Resolution {
pub fn iter(&self) -> impl Iterator<Item = (&PackageName, &Version)> {
self.0.iter()
}
}
/// Resolve a set of requirements into a set of pinned versions.
pub async fn resolve(
requirements: &Requirements,
python_version: &Version,
markers: &MarkerEnvironment,
platform: &Platform,
cache: Option<&Path>,
) -> Result<Resolution> {
// Instantiate a client.
let pypi_client = {
let mut pypi_client = PypiClientBuilder::default();
if let Some(cache) = cache {
pypi_client = pypi_client.cache(cache);
}
pypi_client.build()
};
// A channel to fetch package metadata (e.g., given `flask`, fetch all versions) and version
// metadata (e.g., given `flask==1.0.0`, fetch the metadata for that version).
let (package_sink, package_stream) = futures::channel::mpsc::unbounded();
// Initialize the package stream.
let mut package_stream = package_stream
.map(|request: Request| match request {
Request::Package(requirement) => Either::Left(
pypi_client
.simple(requirement.name.clone())
.map_ok(move |metadata| Response::Package(metadata, requirement)),
),
Request::Version(requirement, file) => Either::Right(
pypi_client
.file(file)
.map_ok(move |metadata| Response::Version(metadata, requirement)),
),
})
.buffer_unordered(32)
.ready_chunks(32);
// Push all the requirements into the package sink.
let mut in_flight: HashSet<PackageName> = HashSet::with_capacity(requirements.len());
for requirement in requirements.iter() {
debug!("--> adding root dependency: {}", requirement);
package_sink.unbounded_send(Request::Package(requirement.clone()))?;
in_flight.insert(PackageName::normalize(&requirement.name));
}
// Determine the compatible platform tags.
let tags = platform.compatible_tags(python_version)?;
// Resolve the requirements.
let mut resolution: HashMap<PackageName, Version> = HashMap::with_capacity(requirements.len());
while let Some(chunk) = package_stream.next().await {
for result in chunk {
let result: Response = result?;
match result {
Response::Package(metadata, requirement) => {
// TODO(charlie): Support URLs. Right now, we treat a URL as an unpinned dependency.
let specifiers =
requirement
.version_or_url
.as_ref()
.and_then(|version_or_url| match version_or_url {
VersionOrUrl::VersionSpecifier(specifiers) => Some(specifiers),
VersionOrUrl::Url(_) => None,
});
// Pick a version that satisfies the requirement.
let Some(file) = metadata.files.iter().rev().find(|file| {
// We only support wheels for now.
let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else {
return false;
};
let Ok(version) = Version::from_str(&name.version) else {
return false;
};
if !name.is_compatible(&tags) {
return false;
}
specifiers
.iter()
.all(|specifier| specifier.contains(&version))
}) else {
continue;
};
package_sink.unbounded_send(Request::Version(requirement, file.clone()))?;
}
Response::Version(metadata, requirement) => {
debug!(
"--> selected version {} for {}",
metadata.version, requirement
);
// Add to the resolved set.
let normalized_name = PackageName::normalize(&requirement.name);
in_flight.remove(&normalized_name);
resolution.insert(normalized_name, metadata.version);
// Enqueue its dependencies.
for dependency in metadata.requires_dist {
if !dependency.evaluate_markers(
markers,
requirement.extras.clone().unwrap_or_default(),
) {
debug!("--> ignoring {dependency} due to environment mismatch");
continue;
}
let normalized_name = PackageName::normalize(&dependency.name);
if resolution.contains_key(&normalized_name) {
continue;
}
if !in_flight.insert(normalized_name) {
continue;
}
debug!("--> adding transitive dependency: {}", dependency);
package_sink.unbounded_send(Request::Package(dependency))?;
}
}
}
}
if in_flight.is_empty() {
break;
}
}
Ok(Resolution(resolution))
}
#[derive(Debug)]
enum Request {
Package(Requirement),
Version(Requirement, File),
}
#[derive(Debug)]
enum Response {
Package(SimpleJson, Requirement),
Version(Metadata21, Requirement),
}