mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Remove resolve_cli.rs and resolve_many.rs (#3606)

Browse source 
## Summary

These depend on some APIs that I want to be internal-only for the
resolver crate, and we're no longer using them.
This commit is contained in:
Charlie Marsh 2024-05-15 12:04:47 -04:00 • committed by GitHub
parent 913fc91afe
commit d76b023cd6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 5 additions and 420 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Cargo.lock generated
View file

@ -4703,14 +4703,11 @@ dependencies = [
"distribution-types",
"fs-err",
"futures",
"indicatif",
"install-wheel-rs",
"itertools 0.12.1",
"mimalloc",
"owo-colors",
"pep440_rs",
"pep508_rs",
"petgraph",
"poloto",
"pretty_assertions",
"resvg",
@ -4730,11 +4727,9 @@ dependencies = [
"uv-client",
"uv-configuration",
"uv-dispatch",
"uv-distribution",
"uv-fs",
"uv-installer",
"uv-interpreter",
"uv-normalize",
"uv-requirements",
"uv-resolver",
"uv-types",

5
crates/uv-dev/Cargo.toml
View file

@ -19,18 +19,15 @@ workspace = true
distribution-filename = { workspace = true }
distribution-types = { workspace = true }
install-wheel-rs = { workspace = true }
pep440_rs = { workspace = true }
pep508_rs = { workspace = true }
uv-build = { workspace = true }
uv-cache = { workspace = true, features = ["clap"] }
uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-dispatch = { workspace = true }
uv-distribution = { workspace = true }
uv-fs = { workspace = true }
uv-installer = { workspace = true }
uv-interpreter = { workspace = true }
uv-normalize = { workspace = true }
uv-requirements = { workspace = true, features = ["schemars"] }
uv-resolver = { workspace = true }
uv-types = { workspace = true }
@ -43,10 +40,8 @@ anyhow = { workspace = true }
clap = { workspace = true, features = ["derive", "wrap_help"] }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
indicatif = { workspace = true }
itertools = { workspace = true }
owo-colors = { workspace = true }
petgraph = { workspace = true }
poloto = { version = "19.1.2" }
pretty_assertions = { version = "1.4.0" }
resvg = { version = "0.29.0" }

30
crates/uv-dev/src/main.rs
View file

@ -16,15 +16,12 @@ use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use resolve_many::ResolveManyArgs;
use crate::build::{build, BuildArgs};
use crate::clear_compile::ClearCompileArgs;
use crate::compile::CompileArgs;
use crate::fetch_python::FetchPythonArgs;
use crate::generate_json_schema::GenerateJsonSchemaArgs;
use crate::render_benchmarks::RenderBenchmarksArgs;
use crate::resolve_cli::ResolveCliArgs;
use crate::wheel_metadata::WheelMetadataArgs;
#[cfg(target_os = "windows")]
@ -49,36 +46,23 @@ mod compile;
mod fetch_python;
mod generate_json_schema;
mod render_benchmarks;
mod resolve_cli;
mod resolve_many;
mod wheel_metadata;
const ROOT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");
#[derive(Parser)]
enum Cli {
/// Build a source distribution into a wheel
/// Build a source distribution into a wheel.
Build(BuildArgs),
/// Resolve many requirements independently in parallel and report failures and successes.
///
/// Run `scripts/popular_packages/pypi_8k_downloads.sh` once, then
/// ```bash
/// cargo run --bin uv-dev -- resolve-many scripts/popular_packages/pypi_8k_downloads.txt
/// ```
/// or
/// ```bash
/// cargo run --bin uv-dev -- resolve-many scripts/popular_packages/pypi_10k_most_dependents.txt
/// ```
ResolveMany(ResolveManyArgs),
/// Resolve requirements passed on the CLI
Resolve(ResolveCliArgs),
/// Display the metadata for a `.whl` at a given URL.
WheelMetadata(WheelMetadataArgs),
/// Render the benchmarks.
RenderBenchmarks(RenderBenchmarksArgs),
/// Compile all `.py` to `.pyc` files in the tree.
Compile(CompileArgs),
/// Remove all `.pyc` in the tree.
ClearCompile(ClearCompileArgs),
/// Fetch Python versions for testing
/// Fetch Python versions for testing.
FetchPython(FetchPythonArgs),
/// Generate JSON schema for the TOML configuration file.
GenerateJSONSchema(GenerateJsonSchemaArgs),
@ -92,12 +76,6 @@ async fn run() -> Result<()> {
let target = build(args).await?;
println!("Wheel built to {}", target.display());
}
Cli::ResolveMany(args) => {
resolve_many::resolve_many(args).await?;
}
Cli::Resolve(args) => {
resolve_cli::resolve_cli(args).await?;
}
Cli::WheelMetadata(args) => wheel_metadata::wheel_metadata(args).await?,
Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?,
Cli::Compile(args) => compile::compile(args).await?,

162
crates/uv-dev/src/resolve_cli.rs
View file

@ -1,162 +0,0 @@
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use anstream::println;
use anyhow::{Context, Result};
use clap::{Parser, ValueEnum};
use fs_err::File;
use itertools::Itertools;
use petgraph::dot::{Config as DotConfig, Dot};
use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Requirement, Resolution};
use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{Concurrency, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
use uv_installer::SitePackages;
use uv_interpreter::PythonEnvironment;
use uv_resolver::{
ExcludeNewer, FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, Resolver,
};
use uv_types::{BuildIsolation, HashStrategy, InFlight};
#[derive(ValueEnum, Default, Clone)]
pub(crate) enum ResolveCliFormat {
#[default]
Compact,
Expanded,
}
#[derive(Parser)]
pub(crate) struct ResolveCliArgs {
requirements: Vec<pep508_rs::Requirement>,
/// Write debug output in DOT format for graphviz to this file
#[clap(long)]
graphviz: Option<PathBuf>,
/// Don't build source distributions. This means resolving will not run arbitrary code. The
/// cached wheels of already built source distributions will be reused.
#[clap(long)]
no_build: bool,
#[clap(long, default_value = "compact")]
format: ResolveCliFormat,
#[command(flatten)]
cache_args: CacheArgs,
#[arg(long)]
exclude_newer: Option<ExcludeNewer>,
#[clap(long, short, env = "UV_INDEX_URL")]
index_url: Option<IndexUrl>,
#[clap(long, env = "UV_EXTRA_INDEX_URL")]
extra_index_url: Vec<IndexUrl>,
#[clap(long)]
find_links: Vec<FlatIndexLocation>,
}
pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
let cache = Cache::try_from(args.cache_args)?;
let venv = PythonEnvironment::from_virtualenv(&cache)?;
let index_locations =
IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false);
let index = InMemoryIndex::default();
let in_flight = InFlight::default();
let no_build = if args.no_build {
NoBuild::All
} else {
NoBuild::None
};
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(
entries,
venv.interpreter().tags()?,
&HashStrategy::None,
&no_build,
&NoBinary::None,
)
};
let config_settings = ConfigSettings::default();
let concurrency = Concurrency::default();
let build_dispatch = BuildDispatch::new(
&client,
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
&index,
&in_flight,
SetupPyStrategy::default(),
&config_settings,
BuildIsolation::Isolated,
install_wheel_rs::linker::LinkMode::default(),
&no_build,
&NoBinary::None,
concurrency,
);
let site_packages = SitePackages::from_executable(&venv)?;
// Copied from `BuildDispatch`
let tags = venv.interpreter().tags()?;
let markers = venv.interpreter().markers();
let python_requirement =
PythonRequirement::from_marker_environment(venv.interpreter(), markers);
let resolver = Resolver::new(
Manifest::simple(
args.requirements
.iter()
.cloned()
.map(Requirement::from_pep508)
.collect::<Result<_, _>>()?,
),
Options::default(),
&python_requirement,
Some(venv.interpreter().markers()),
tags,
&flat_index,
&index,
&HashStrategy::None,
&build_dispatch,
&site_packages,
DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads),
)?;
let resolution_graph = resolver.resolve().await.with_context(|| {
format!(
"No solution found when resolving: {}",
args.requirements.iter().map(ToString::to_string).join(", "),
)
})?;
if let Some(graphviz) = args.graphviz {
let mut writer = BufWriter::new(File::create(graphviz)?);
let graphviz = Dot::with_attr_getters(
resolution_graph.petgraph(),
&[DotConfig::NodeNoLabel, DotConfig::EdgeNoLabel],
&|_graph, edge_ref| format!("label={:?}", edge_ref.weight().to_string()),
&|_graph, (_node_index, dist)| {
format!("label={:?}", dist.to_string().replace("==", "\n"))
},
);
write!(&mut writer, "{graphviz:?}")?;
}
let requirements = Resolution::from(resolution_graph).requirements();
match args.format {
ResolveCliFormat::Compact => {
println!("{}", requirements.iter().map(ToString::to_string).join(" "));
}
ResolveCliFormat::Expanded => {
for package in requirements {
println!("{}", package);
}
}
}
Ok(())
}

214
crates/uv-dev/src/resolve_many.rs
View file

@ -1,214 +0,0 @@
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::Result;
use clap::Parser;
use futures::StreamExt;
use indicatif::ProgressStyle;
use itertools::Itertools;
use tokio::time::Instant;
use tracing::{info, info_span, Span};
use tracing_indicatif::span_ext::IndicatifSpanExt;
use distribution_types::{IndexLocations, Requirement};
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::VersionOrUrl;
use uv_cache::{Cache, CacheArgs};
use uv_client::{OwnedArchive, RegistryClient, RegistryClientBuilder};
use uv_configuration::{Concurrency, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment;
use uv_normalize::PackageName;
use uv_resolver::{FlatIndex, InMemoryIndex};
use uv_types::{BuildContext, BuildIsolation, InFlight};
#[derive(Parser)]
pub(crate) struct ResolveManyArgs {
/// Path to a file containing one requirement per line.
requirements: PathBuf,
#[clap(long)]
limit: Option<usize>,
/// Don't build source distributions. This means resolving will not run arbitrary code. The
/// cached wheels of already built source distributions will be reused.
#[clap(long)]
no_build: bool,
/// Run this many tasks in parallel.
#[clap(long, default_value = "50")]
num_tasks: usize,
/// Force the latest version when no version is given.
#[clap(long)]
latest_version: bool,
#[command(flatten)]
cache_args: CacheArgs,
}
/// Try to find the latest version of a package, ignoring error because we report them during resolution properly
async fn find_latest_version(
client: &RegistryClient,
package_name: &PackageName,
) -> Option<Version> {
client
.simple(package_name)
.await
.ok()
.into_iter()
.flatten()
.filter_map(|(_index, raw_simple_metadata)| {
let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata);
Some(simple_metadata.into_iter().next()?.version)
})
.max()
}
pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {
let cache = Cache::try_from(args.cache_args)?;
let data = fs_err::read_to_string(&args.requirements)?;
let tf_models_nightly = PackageName::from_str("tf-models-nightly").unwrap();
let lines = data
.lines()
.map(pep508_rs::Requirement::from_str)
.filter_ok(|req| req.name != tf_models_nightly);
let requirements: Vec<pep508_rs::Requirement> = if let Some(limit) = args.limit {
lines.take(limit).collect::<Result<_, _>>()?
} else {
lines.collect::<Result<_, _>>()?
};
let total = requirements.len();
let venv = PythonEnvironment::from_virtualenv(&cache)?;
let in_flight = InFlight::default();
let concurrency = Concurrency::default();
let client = RegistryClientBuilder::new(cache.clone()).build();
let header_span = info_span!("resolve many");
header_span.pb_set_style(&ProgressStyle::default_bar());
header_span.pb_set_length(total as u64);
let _header_span_enter = header_span.enter();
let no_build = if args.no_build {
NoBuild::All
} else {
NoBuild::None
};
let mut tasks = futures::stream::iter(requirements)
.map(|requirement| {
async {
// Use a separate `InMemoryIndex` for each requirement.
let index = InMemoryIndex::default();
let index_locations = IndexLocations::default();
let setup_py = SetupPyStrategy::default();
let flat_index = FlatIndex::default();
let config_settings = ConfigSettings::default();
// Create a `BuildDispatch` for each requirement.
let build_dispatch = BuildDispatch::new(
&client,
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
&index,
&in_flight,
setup_py,
&config_settings,
BuildIsolation::Isolated,
install_wheel_rs::linker::LinkMode::default(),
&no_build,
&NoBinary::None,
concurrency,
);
let start = Instant::now();
let requirement = if args.latest_version && requirement.version_or_url.is_none() {
if let Some(version) = find_latest_version(&client, &requirement.name).await {
let equals_version = VersionOrUrl::VersionSpecifier(
VersionSpecifiers::from(VersionSpecifier::equals_version(version)),
);
pep508_rs::Requirement {
name: requirement.name,
extras: requirement.extras,
version_or_url: Some(equals_version),
marker: None,
origin: requirement.origin,
}
} else {
requirement
}
} else {
requirement
};
let result = build_dispatch
.resolve(&[
Requirement::from_pep508(requirement.clone()).expect("Invalid requirement")
])
.await;
(requirement.to_string(), start.elapsed(), result)
}
})
.buffer_unordered(args.num_tasks);
let mut success = 0usize;
let mut errors = Vec::new();
while let Some(result) = tasks.next().await {
let (package, duration, result) = result;
match result {
Ok(_) => {
info!(
"Success ({}/{}, {} ms): {}",
success + errors.len(),
total,
duration.as_millis(),
package,
);
success += 1;
}
Err(err) => {
let err_formatted =
if err
.source()
.and_then(|err| err.source())
.is_some_and(|err| {
err.to_string() == "Building source distributions is disabled"
})
{
"Building source distributions is disabled".to_string()
} else {
err.chain()
.map(|err| {
let formatted = err.to_string();
// Cut overly long c/c++ compile output
if formatted.lines().count() > 20 {
let formatted: Vec<_> = formatted.lines().collect();
formatted[..20].join("\n")
+ "\n[...]\n"
+ &formatted[formatted.len() - 20..].join("\n")
} else {
formatted
}
})
.join("\n Caused by: ")
};
info!(
"Error for {} ({}/{}, {} ms): {}",
package,
success + errors.len(),
total,
duration.as_millis(),
err_formatted
);
errors.push(package);
}
}
Span::current().pb_inc(1);
}
info!("Errors: {}", errors.join(", "));
info!("Success: {}, Error: {}", success, errors.len());
Ok(())
}

7
crates/uv-resolver/src/resolution/graph.rs
View file

@ -376,13 +376,6 @@ impl ResolutionGraph {
&self.diagnostics
}
/// Return the underlying graph.
pub fn petgraph(
&self,
) -> &petgraph::graph::Graph<AnnotatedDist, Range<Version>, petgraph::Directed> {
&self.petgraph
}
/// Return the marker tree specific to this resolution.
///
/// This accepts a manifest, in-memory-index and marker environment. All

2
crates/uv-resolver/src/resolution/mod.rs
View file

@ -14,7 +14,7 @@ mod graph;
/// specific distribution (e.g., a specific wheel), while the [`Metadata23`] refers to the metadata
/// for the package-version pair.
#[derive(Debug)]
pub struct AnnotatedDist {
pub(crate) struct AnnotatedDist {
pub(crate) dist: ResolvedDist,
pub(crate) extras: Vec<ExtraName>,
pub(crate) hashes: Vec<HashDigest>,