mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Use Resolver in pip sync (#2696)

Browse source 
## Summary

This PR removes the custom `DistFinder` that we use in `pip sync`. This
originally existed because `VersionMap` wasn't lazy, and so we saved a
lot of time in `DistFinder` by reading distribution data lazily. But
now, AFAICT, there's really no benefit. Maintaining `DistFinder` means
we effectively have to maintain two resolvers, and end up fixing bugs in
`DistFinder` that don't exist in the `Resolver` (like #2688.

Closes #2694.

Closes #2443.

## Test Plan

I ran this benchmark a bunch. It's basically a wash. Sometimes one is
faster than the other.

```
❯ python -m scripts.bench \
        --uv-path ./target/release/main \
        --uv-path ./target/release/uv \
        scripts/requirements/compiled/trio.txt --min-runs 50 --benchmark install-warm --warmup 25
Benchmark 1: ./target/release/main (install-warm)
  Time (mean ± σ):      54.0 ms ±  10.6 ms    [User: 8.7 ms, System: 98.1 ms]
  Range (min … max):    45.5 ms …  94.3 ms    50 runs

  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet PC without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.

Benchmark 2: ./target/release/uv (install-warm)
  Time (mean ± σ):      50.7 ms ±   9.2 ms    [User: 8.7 ms, System: 98.6 ms]
  Range (min … max):    44.0 ms …  98.6 ms    50 runs

  Warning: The first benchmarking run for this command was significantly slower than the rest (77.6 ms). This could be caused by (filesystem) caches that were not filled until after the first run. You should consider using the '--warmup' option to fill those caches before the actual benchmark. Alternatively, use the '--prepare' option to clear the caches before each timing run.

Summary
  './target/release/uv (install-warm)' ran
    1.06 ± 0.29 times faster than './target/release/main (install-warm)'
```
This commit is contained in:
Charlie Marsh 2024-03-27 17:36:38 -04:00 • committed by GitHub
parent d41ab0ef4d
commit f8fa887c0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 64 additions and 570 deletions
Download patch file Download diff file Expand all files Collapse all files

195
crates/uv-dev/src/install_many.rs
View file

@ -1,195 +0,0 @@
use std::path::PathBuf;
use std::str::FromStr;
use anstream::eprintln;
use anyhow::{Context, Result};
use clap::Parser;
use futures::StreamExt;
use itertools::{Either, Itertools};
use rustc_hash::FxHashMap;
use tracing::info;
use distribution_types::{
CachedDist, Dist, DistributionMetadata, IndexLocations, Name, Resolution, VersionOrUrl,
};
use install_wheel_rs::linker::LinkMode;
use pep508_rs::Requirement;
use platform_tags::Tags;
use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, RegistryClient, RegistryClientBuilder};
use uv_dispatch::BuildDispatch;
use uv_distribution::RegistryWheelIndex;
use uv_installer::{Downloader, NoBinary};
use uv_interpreter::PythonEnvironment;
use uv_normalize::PackageName;
use uv_resolver::{DistFinder, InMemoryIndex};
use uv_types::{BuildContext, BuildIsolation, ConfigSettings, InFlight, NoBuild, SetupPyStrategy};
#[derive(Parser)]
pub(crate) struct InstallManyArgs {
/// Path to a file containing one requirement per line.
requirements: PathBuf,
#[clap(long)]
limit: Option<usize>,
/// Don't build source distributions. This means resolving will not run arbitrary code. The
/// cached wheels of already built source distributions will be reused.
#[clap(long)]
no_build: bool,
/// Run this many tasks in parallel
#[clap(long, default_value = "50")]
num_tasks: usize,
#[command(flatten)]
cache_args: CacheArgs,
}
pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> {
let data = fs_err::read_to_string(&args.requirements)?;
let lines = data.lines().map(Requirement::from_str);
let requirements: Vec<Requirement> = if let Some(limit) = args.limit {
lines.take(limit).collect::<Result<_, _>>()?
} else {
lines.collect::<Result<_, _>>()?
};
info!("Got {} requirements", requirements.len());
let cache = Cache::try_from(args.cache_args)?;
let venv = PythonEnvironment::from_virtualenv(&cache)?;
let client = RegistryClientBuilder::new(cache.clone()).build();
let index_locations = IndexLocations::default();
let flat_index = FlatIndex::default();
let index = InMemoryIndex::default();
let setup_py = SetupPyStrategy::default();
let in_flight = InFlight::default();
let tags = venv.interpreter().tags()?;
let no_build = if args.no_build {
NoBuild::All
} else {
NoBuild::None
};
let config_settings = ConfigSettings::default();
let build_dispatch = BuildDispatch::new(
&client,
&cache,
venv.interpreter(),
&index_locations,
&flat_index,
&index,
&in_flight,
setup_py,
&config_settings,
BuildIsolation::Isolated,
&no_build,
&NoBinary::None,
);
for (idx, requirements) in requirements.chunks(100).enumerate() {
info!("Chunk {idx}");
if let Err(err) = install_chunk(
requirements,
&build_dispatch,
tags,
&client,
&venv,
&index_locations,
)
.await
{
eprintln!("💥 Chunk {idx} failed");
for cause in err.chain() {
eprintln!(" Caused by: {cause}");
}
}
}
Ok(())
}
async fn install_chunk(
requirements: &[Requirement],
build_dispatch: &BuildDispatch<'_>,
tags: &Tags,
client: &RegistryClient,
venv: &PythonEnvironment,
index_locations: &IndexLocations,
) -> Result<()> {
let resolution: Vec<_> = DistFinder::new(
tags,
client,
venv.interpreter(),
&FlatIndex::default(),
&NoBinary::None,
&NoBuild::None,
)
.resolve_stream(requirements)
.collect()
.await;
let (resolution, failures): (FxHashMap<PackageName, Dist>, Vec<_>) =
resolution.into_iter().partition_result();
for failure in &failures {
info!("Failed to find wheel: {failure}");
}
if !failures.is_empty() {
info!("Failed to find {} wheel(s)", failures.len());
}
let wheels_and_source_dist = resolution.len();
let resolution = if build_dispatch.no_build().is_none() {
resolution
} else {
let only_wheels: FxHashMap<_, _> = resolution
.into_iter()
.filter(|(_, dist)| match dist {
Dist::Built(_) => true,
Dist::Source(_) => false,
})
.collect();
info!(
"Removed {} source dists",
wheels_and_source_dist - only_wheels.len()
);
only_wheels
};
let dists = Resolution::new(resolution)
.into_distributions()
.collect::<Vec<_>>();
let mut registry_index = RegistryWheelIndex::new(build_dispatch.cache(), tags, index_locations);
let (cached, uncached): (Vec<_>, Vec<_>) = dists.iter().partition_map(|dist| {
// We always want the wheel for the latest version not whatever matching is in cache.
let VersionOrUrl::Version(version) = dist.version_or_url() else {
unreachable!("Only registry distributions are supported");
};
if let Some(cached) = registry_index.get_version(dist.name(), version) {
Either::Left(CachedDist::Registry(cached.clone()))
} else {
Either::Right(dist.clone())
}
});
info!("Cached: {}, Uncached {}", cached.len(), uncached.len());
let downloader = Downloader::new(build_dispatch.cache(), tags, client, build_dispatch);
let in_flight = InFlight::default();
let fetches: Vec<_> = futures::stream::iter(uncached)
.map(|dist| downloader.get_wheel(dist, &in_flight))
.buffer_unordered(50)
.collect()
.await;
let (wheels, failures): (Vec<_>, Vec<_>) = fetches.into_iter().partition_result();
for failure in &failures {
info!("Failed to fetch wheel: {failure}");
}
if !failures.is_empty() {
info!("Failed to fetch {} wheel(s)", failures.len());
}
let wheels: Vec<_> = wheels.into_iter().chain(cached).collect();
uv_installer::Installer::new(venv)
.with_link_mode(LinkMode::default())
.install(&wheels)
.context("Failed to install")?;
info!("Installed {} wheels", wheels.len());
Ok(())
}

6
crates/uv-dev/src/main.rs
View file

@ -21,7 +21,6 @@ use resolve_many::ResolveManyArgs;
use crate::build::{build, BuildArgs};
use crate::clear_compile::ClearCompileArgs;
use crate::compile::CompileArgs;
use crate::install_many::InstallManyArgs;
use crate::render_benchmarks::RenderBenchmarksArgs;
use crate::resolve_cli::ResolveCliArgs;
use crate::wheel_metadata::WheelMetadataArgs;
@ -45,7 +44,6 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
mod build;
mod clear_compile;
mod compile;
mod install_many;
mod render_benchmarks;
mod resolve_cli;
mod resolve_many;
@ -66,7 +64,6 @@ enum Cli {
/// cargo run --bin uv-dev -- resolve-many scripts/popular_packages/pypi_10k_most_dependents.txt
/// ```
ResolveMany(ResolveManyArgs),
InstallMany(InstallManyArgs),
/// Resolve requirements passed on the CLI
Resolve(ResolveCliArgs),
WheelMetadata(WheelMetadataArgs),
@ -88,9 +85,6 @@ async fn run() -> Result<()> {
Cli::ResolveMany(args) => {
resolve_many::resolve_many(args).await?;
}
Cli::InstallMany(args) => {
install_many::install_many(args).await?;
}
Cli::Resolve(args) => {
resolve_cli::resolve_cli(args).await?;
}