From 2ac562b40d63428d18a0c90c587455d24997f55c Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 4 Apr 2024 22:00:39 -0400 Subject: [PATCH] Respect `--no-build` and `--no-binary` in `--find-links` (#2826) ## Summary In working on `--require-hashes`, I noticed that we're missing some incompatibility tracking for `--find-links` distributions. Specifically, we don't respect `--no-build` or `--no-binary`, so if we select a wheel due to `--find-links`, we then throw a hard error when trying to build it later (if `--no-binary` is provided), rather than selecting the source distribution instead. Closes https://github.com/astral-sh/uv/issues/2827. --- crates/uv-client/src/flat_index.rs | 90 +++++++++++++----- crates/uv-dev/src/resolve_cli.rs | 21 ++-- crates/uv/src/commands/pip_compile.rs | 2 +- crates/uv/src/commands/pip_install.rs | 2 +- crates/uv/src/commands/pip_sync.rs | 2 +- crates/uv/src/commands/venv.rs | 2 +- crates/uv/tests/pip_compile.rs | 2 +- crates/uv/tests/pip_install.rs | 64 ++++++++++++- crates/uv/tests/pip_sync.rs | 8 +- .../maturin-1.4.0-py3-none-any.whl | Bin .../maturin-2.0.0-py3-none-linux_x86_64.whl | Bin .../simple_launcher-0.1.0-py3-none-any.whl | Bin .../tqdm-1000.0.0-py3-none-any.whl | Bin ...ylinux2010_x86_64.musllinux_1_1_x86_64.whl | Bin scripts/links/tqdm-999.0.0.tar.gz | Bin 0 -> 2127 bytes 15 files changed, 150 insertions(+), 43 deletions(-) rename scripts/{wheels => links}/maturin-1.4.0-py3-none-any.whl (100%) rename scripts/{wheels => links}/maturin-2.0.0-py3-none-linux_x86_64.whl (100%) rename scripts/{wheels => links}/simple_launcher-0.1.0-py3-none-any.whl (100%) rename scripts/{wheels => links}/tqdm-1000.0.0-py3-none-any.whl (100%) rename scripts/{wheels => links}/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl (100%) create mode 100644 scripts/links/tqdm-999.0.0.tar.gz diff --git a/crates/uv-client/src/flat_index.rs b/crates/uv-client/src/flat_index.rs index a042f4f10..45eac5af8 100644 --- a/crates/uv-client/src/flat_index.rs +++ b/crates/uv-client/src/flat_index.rs @@ -8,10 +8,11 @@ use rustc_hash::FxHashMap; use tracing::{debug, info_span, instrument, warn, Instrument}; use url::Url; -use distribution_filename::DistFilename; +use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use distribution_types::{ - BuiltDist, Dist, File, FileLocation, FlatIndexLocation, IndexUrl, PrioritizedDist, - RegistryBuiltDist, RegistrySourceDist, SourceDist, SourceDistCompatibility, + BuiltDist, Dist, File, FileLocation, FlatIndexLocation, IncompatibleSource, IncompatibleWheel, + IndexUrl, PrioritizedDist, RegistryBuiltDist, RegistrySourceDist, SourceDist, + SourceDistCompatibility, WheelCompatibility, }; use pep440_rs::Version; use pep508_rs::VerbatimUrl; @@ -19,6 +20,7 @@ use platform_tags::Tags; use pypi_types::Hashes; use uv_cache::{Cache, CacheBucket}; use uv_normalize::PackageName; +use uv_types::{NoBinary, NoBuild}; use crate::cached_client::{CacheControl, CachedClientError}; use crate::html::SimpleHtml; @@ -271,12 +273,25 @@ pub struct FlatIndex { impl FlatIndex { /// Collect all files from a `--find-links` target into a [`FlatIndex`]. #[instrument(skip_all)] - pub fn from_entries(entries: FlatIndexEntries, tags: &Tags) -> Self { + pub fn from_entries( + entries: FlatIndexEntries, + tags: &Tags, + no_build: &NoBuild, + no_binary: &NoBinary, + ) -> Self { // Collect compatible distributions. let mut index = FxHashMap::default(); for (filename, file, url) in entries.entries { let distributions = index.entry(filename.name().clone()).or_default(); - Self::add_file(distributions, file, filename, tags, url); + Self::add_file( + distributions, + file, + filename, + tags, + no_build, + no_binary, + url, + ); } // Collect offline entries. @@ -290,15 +305,17 @@ impl FlatIndex { file: File, filename: DistFilename, tags: &Tags, + no_build: &NoBuild, + no_binary: &NoBinary, index: IndexUrl, ) { // No `requires-python` here: for source distributions, we don't have that information; // for wheels, we read it lazily only when selected. match filename { DistFilename::WheelFilename(filename) => { - let compatibility = filename.compatibility(tags); let version = filename.version.clone(); + let compatibility = Self::wheel_compatibility(&filename, tags, no_binary); let dist = Dist::Built(BuiltDist::Registry(RegistryBuiltDist { filename, file: Box::new(file), @@ -306,20 +323,15 @@ impl FlatIndex { })); match distributions.0.entry(version) { Entry::Occupied(mut entry) => { - entry - .get_mut() - .insert_built(dist, None, compatibility.into()); + entry.get_mut().insert_built(dist, None, compatibility); } Entry::Vacant(entry) => { - entry.insert(PrioritizedDist::from_built( - dist, - None, - compatibility.into(), - )); + entry.insert(PrioritizedDist::from_built(dist, None, compatibility)); } } } DistFilename::SourceDistFilename(filename) => { + let compatibility = Self::source_dist_compatibility(&filename, no_build); let dist = Dist::Source(SourceDist::Registry(RegistrySourceDist { filename: filename.clone(), file: Box::new(file), @@ -327,24 +339,54 @@ impl FlatIndex { })); match distributions.0.entry(filename.version) { Entry::Occupied(mut entry) => { - entry.get_mut().insert_source( - dist, - None, - SourceDistCompatibility::Compatible, - ); + entry.get_mut().insert_source(dist, None, compatibility); } Entry::Vacant(entry) => { - entry.insert(PrioritizedDist::from_source( - dist, - None, - SourceDistCompatibility::Compatible, - )); + entry.insert(PrioritizedDist::from_source(dist, None, compatibility)); } } } } } + fn source_dist_compatibility( + filename: &SourceDistFilename, + no_build: &NoBuild, + ) -> SourceDistCompatibility { + // Check if source distributions are allowed for this package. + let no_build = match no_build { + NoBuild::None => false, + NoBuild::All => true, + NoBuild::Packages(packages) => packages.contains(&filename.name), + }; + + if no_build { + return SourceDistCompatibility::Incompatible(IncompatibleSource::NoBuild); + } + + SourceDistCompatibility::Compatible + } + + fn wheel_compatibility( + filename: &WheelFilename, + tags: &Tags, + no_binary: &NoBinary, + ) -> WheelCompatibility { + // Check if binaries are allowed for this package. + let no_binary = match no_binary { + NoBinary::None => false, + NoBinary::All => true, + NoBinary::Packages(packages) => packages.contains(&filename.name), + }; + + if no_binary { + return WheelCompatibility::Incompatible(IncompatibleWheel::NoBinary); + } + + // Determine a compatibility for the wheel based on tags. + WheelCompatibility::from(filename.compatibility(tags)) + } + /// Get the [`FlatDistributions`] for the given package name. pub fn get(&self, package_name: &PackageName) -> Option<&FlatDistributions> { self.index.get(package_name) diff --git a/crates/uv-dev/src/resolve_cli.rs b/crates/uv-dev/src/resolve_cli.rs index bb320628c..3d9bb3cc7 100644 --- a/crates/uv-dev/src/resolve_cli.rs +++ b/crates/uv-dev/src/resolve_cli.rs @@ -56,14 +56,6 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { let venv = PythonEnvironment::from_virtualenv(&cache)?; let index_locations = IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false); - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_locations.index_urls()) - .build(); - let flat_index = { - let client = FlatIndexClient::new(&client, &cache); - let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries(entries, venv.interpreter().tags()?) - }; let index = InMemoryIndex::default(); let in_flight = InFlight::default(); let no_build = if args.no_build { @@ -71,6 +63,19 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { } else { NoBuild::None }; + let client = RegistryClientBuilder::new(cache.clone()) + .index_urls(index_locations.index_urls()) + .build(); + let flat_index = { + let client = FlatIndexClient::new(&client, &cache); + let entries = client.fetch(index_locations.flat_index()).await?; + FlatIndex::from_entries( + entries, + venv.interpreter().tags()?, + &no_build, + &NoBinary::None, + ) + }; let config_settings = ConfigSettings::default(); let build_dispatch = BuildDispatch::new( diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 31f6fdd94..55c781921 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -228,7 +228,7 @@ pub(crate) async fn pip_compile( let flat_index = { let client = FlatIndexClient::new(&client, &cache); let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries(entries, &tags) + FlatIndex::from_entries(entries, &tags, &no_build, &NoBinary::None) }; // Track in-flight downloads, builds, etc., across resolutions. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 8e7b398e6..80281a2f3 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -206,7 +206,7 @@ pub(crate) async fn pip_install( let flat_index = { let client = FlatIndexClient::new(&client, &cache); let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries(entries, tags) + FlatIndex::from_entries(entries, tags, &no_build, &no_binary) }; // Determine whether to enable build isolation. diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index c437de4bc..28c10c82f 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -155,7 +155,7 @@ pub(crate) async fn pip_sync( let flat_index = { let client = FlatIndexClient::new(&client, &cache); let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries(entries, tags) + FlatIndex::from_entries(entries, tags, &no_build, &no_binary) }; // Create a shared in-memory index. diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index e6c842cbd..160e7e332 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -169,7 +169,7 @@ async fn venv_impl( .fetch(index_locations.flat_index()) .await .map_err(VenvError::FlatIndex)?; - FlatIndex::from_entries(entries, tags) + FlatIndex::from_entries(entries, tags, &NoBuild::All, &NoBinary::None) }; // Create a shared in-memory index. diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index c4d199483..bf26d9ffb 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -3600,7 +3600,7 @@ fn find_links_directory() -> Result<()> { uv_snapshot!(context.filters(), context.compile() .arg("requirements.in") .arg("--find-links") - .arg(context.workspace_root.join("scripts").join("wheels")), @r###" + .arg(context.workspace_root.join("scripts").join("links")), @r###" success: true exit_code: 0 ----- stdout ----- diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs index 7c2e680f7..acfa3c947 100644 --- a/crates/uv/tests/pip_install.rs +++ b/crates/uv/tests/pip_install.rs @@ -1854,7 +1854,7 @@ fn launcher() -> Result<()> { uv_snapshot!( filters, context.install() - .arg(format!("simple_launcher@{}", project_root.join("scripts/wheels/simple_launcher-0.1.0-py3-none-any.whl").display())) + .arg(format!("simple_launcher@{}", project_root.join("scripts/links/simple_launcher-0.1.0-py3-none-any.whl").display())) .arg("--strict"), @r###" success: true exit_code: 0 @@ -1899,7 +1899,7 @@ fn launcher_with_symlink() -> Result<()> { uv_snapshot!(filters, context.install() - .arg(format!("simple_launcher@{}", project_root.join("scripts/wheels/simple_launcher-0.1.0-py3-none-any.whl").display())) + .arg(format!("simple_launcher@{}", project_root.join("scripts/links/simple_launcher-0.1.0-py3-none-any.whl").display())) .arg("--strict"), @r###" success: true @@ -3739,3 +3739,63 @@ fn already_installed_remote_url() { `) "###); } + +/// Sync using `--find-links` with a local directory. +#[test] +fn find_links() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc! {r" + tqdm + "})?; + + uv_snapshot!(context.filters(), context.install() + .arg("tqdm") + .arg("--find-links") + .arg(context.workspace_root.join("scripts/links/")), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + tqdm==1000.0.0 + "### + ); + + Ok(()) +} + +/// Sync using `--find-links` with a local directory, with wheels disabled. +#[test] +fn find_links_no_binary() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc! {r" + tqdm + "})?; + + uv_snapshot!(context.filters(), context.install() + .arg("tqdm") + .arg("--no-binary") + .arg(":all:") + .arg("--find-links") + .arg(context.workspace_root.join("scripts/links/")), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + tqdm==999.0.0 + "### + ); + + Ok(()) +} diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index b95f001cf..307d425ba 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -2461,7 +2461,7 @@ fn find_links() -> Result<()> { uv_snapshot!(context.filters(), command(&context) .arg("requirements.txt") .arg("--find-links") - .arg(context.workspace_root.join("scripts/wheels/")), @r###" + .arg(context.workspace_root.join("scripts/links/")), @r###" success: true exit_code: 0 ----- stdout ----- @@ -2494,7 +2494,7 @@ fn find_links_no_index_match() -> Result<()> { .arg("requirements.txt") .arg("--no-index") .arg("--find-links") - .arg(context.workspace_root.join("scripts/wheels/")), @r###" + .arg(context.workspace_root.join("scripts/links/")), @r###" success: true exit_code: 0 ----- stdout ----- @@ -2524,7 +2524,7 @@ fn find_links_offline_match() -> Result<()> { .arg("requirements.txt") .arg("--offline") .arg("--find-links") - .arg(context.workspace_root.join("scripts/wheels/")), @r###" + .arg(context.workspace_root.join("scripts/links/")), @r###" success: true exit_code: 0 ----- stdout ----- @@ -2555,7 +2555,7 @@ fn find_links_offline_no_match() -> Result<()> { .arg("requirements.txt") .arg("--offline") .arg("--find-links") - .arg(context.workspace_root.join("scripts/wheels/")), @r###" + .arg(context.workspace_root.join("scripts/links/")), @r###" success: false exit_code: 2 ----- stdout ----- diff --git a/scripts/wheels/maturin-1.4.0-py3-none-any.whl b/scripts/links/maturin-1.4.0-py3-none-any.whl similarity index 100% rename from scripts/wheels/maturin-1.4.0-py3-none-any.whl rename to scripts/links/maturin-1.4.0-py3-none-any.whl diff --git a/scripts/wheels/maturin-2.0.0-py3-none-linux_x86_64.whl b/scripts/links/maturin-2.0.0-py3-none-linux_x86_64.whl similarity index 100% rename from scripts/wheels/maturin-2.0.0-py3-none-linux_x86_64.whl rename to scripts/links/maturin-2.0.0-py3-none-linux_x86_64.whl diff --git a/scripts/wheels/simple_launcher-0.1.0-py3-none-any.whl b/scripts/links/simple_launcher-0.1.0-py3-none-any.whl similarity index 100% rename from scripts/wheels/simple_launcher-0.1.0-py3-none-any.whl rename to scripts/links/simple_launcher-0.1.0-py3-none-any.whl diff --git a/scripts/wheels/tqdm-1000.0.0-py3-none-any.whl b/scripts/links/tqdm-1000.0.0-py3-none-any.whl similarity index 100% rename from scripts/wheels/tqdm-1000.0.0-py3-none-any.whl rename to scripts/links/tqdm-1000.0.0-py3-none-any.whl diff --git a/scripts/wheels/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl b/scripts/links/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl similarity index 100% rename from scripts/wheels/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl rename to scripts/links/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl diff --git a/scripts/links/tqdm-999.0.0.tar.gz b/scripts/links/tqdm-999.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cbe4150bd7071bd5788a33df0a29aa8031df0758 GIT binary patch literal 2127 zcmV-V2(b4biwFP!5jI`||Lq!UPvc0E&;2Xv%!eIN>^#UYM|6xXBxbNCBvBG(7NICO zb|>-S*Sgz4cD4U~tJ}^CW)`xGqTSmrAvxV$uP#@0Rkh(ixq*Iia$=N>Qjt@}S#dP- zLXVF|Mm*np6DT1ZSF7@g@bpfIAN6z3r;R9{C*E|%2lSEGpi-(-^*Eu7h8!9*JMldVmv+Kt@E<1|8i^5z ziT_Rm+w%=43OdjPr+XZ_(uHW0K?xNN9iu(zfyh{ z|CQRkBC#Qz(o|38xd@rR0kmH!p&H2#gL$GvG7 zCG<^YycGYX(uXJUkLACL|DC|t{uvK?6_hIeG5%Xt!)*6WgFo^e#y^(-PxF7}Sk?c# z0nEsdXKF&06#%?*4*qBXK)N&uJSHl9@EFW!LdSEMCU(fFTYyPIDMS+x9Hz7YJOVqM zLySEE5=3KeWBD0Q!3GDl5tw|2Y*;km4|YNk*#(=i$nk9C2X5q~0V-&7A#Z|Z41;%k zMnUW66xx9l;!@iOF9fl+Sb_(S&#>18+$2vtNAM^hlH;eY&|$Icdx4ir5aiNCOfXFu z?m*~SfFN?ciFi`EtvDU~9-9@w^#tp2%5jp3NonhX*heu+fKlHU96Z!p?scV|#1T?r zVNjl%NXmVfMZxAy&_t6o2~i}KG;V|@lM??&9WG`B`XuuG=s|4Fi9**CJ7P6L2DpSx z|2>s^$vhy8II5bdAYMScdaL;oo7t%QnC1pUxoBQ{Z99n&kKu*oAKE^S5-IQLHVu@2 zVM4!iKKNSinPByy+v|L>nr0KUdLPGH0lr#;i_X;m5TjRb53Zqe4)yjme74%n0+>H` zduG26ogP@1-IitIoYii$u9{Z+6PzJkyMu0Sp|c~?K?j7uoRwwv1+z=D*SNr8{mg1v zgX;pETZ6XXd5%o$(5?3dR^zHw??Lyf*X{I86yHR)ZL59WLn-E^*&Y}u73aYGf&=JZ z)LShns(yvq_oRM}PWQTJeYzOHMW@v?aq`SW&FW_@GZTfIYP9OsWdWM?%lao%Qgx6^ zPom}8eZ4T{6w0gPUt?f(+F~1xPJ7V9VF7p98!Sn`T79zs^`6xi208C_ki9S`QgkE} zq-&cQ7h!7Hcq51+##eoFrAyPSw~$v~5U(Q}s+9Qq=Kr2qZ(f>4;J&Slm+JoyrDys7 zsPtat|K0Te!UG7o`PWT%-nBr7FWK$f@ZF5_nAM6!FPIu^=272eh8Gp%dF&PMcaHfqe4T`No?jEPEkqGr03nBm1AFYtI-?O`-4Swj zc!;ZTV}eP1T?;k$irZFnHo^I8a(@=T3P_xE9)jF8*gTg;y<@+pbE7;+|=C2>ts zEQ_jJF?}l%^r{qaAq`!c$UMj~u`ZM14W_8oE?Mk8%3bEVup%xSGamQ{kWc;D^L8Wt z(W;p_4ITyluHyfwdQ^Ve|9_~e{(mQMGfq9<)!CeJ8VpH7e@Z>9hT#-$w3*GFnUB?? zHY6F^*mmwHR)nXpS~TR6hDS*5JVO%N0TnpHxisR=aqKygWggSW(51{tyjT|1r|`oM zDB5;2vSk|5mip*%c68wC$I~O@-x|sNU(8@oXylI0AELySnh%McB6=bR1mG7~XtdGB zU|mT?-y#;i+t6t6Yk>@j<73kyZ!s54wI;o%eiV!D1Nzt=qzq~`sNy+yM!BsO1Wq@J zu=5E5QF6EJaGKgvDzLJX@HIH%m)P*DQBPivS}tvYTiyb<@>*Ogh<&>1-*Bd}ooSvk zz+5xe#4B@YjJ`@k$D_GdEM3${6Q2!9bEA0_PMsgTQ*8Xwv3UK(G`tI=hfpS%{me%- zVlkFVrY}(xJ&CE2ev~E-eFcRGBpyY+Ap^|F17=8Ko6i<8wegMi2>}i7nc>ojo%;N_ z4TMo0Z1fT|uY%A=Gne5 zZ)3z`T7Dd&jTeW^jV415G#O6fAwoqy%lWgS4ZmS{K94E;w%LPuJpXuFmLEI6X<|vc zGO#nF&YduzAOu~{s|;PD;W&&!oWttDcDRSFdu~v44_H_=m$5LJfp1_)nDoR^d^gQb zdFgb8%yLYe^oboh!qs?^Vtr#LD3tn(aU^ac7PfxoikXo&g$la*{^mGV(Z)qgv=|HV?pwYjb5O<4^p z#t~^_wA8YGW;vfb3F&4jRKM!AYOs+(3-G%9xyd5Sr7zOvnRJsj%c?KZlDaGhby@Az zwz$h)e8V;$#XWdt2k{MiF)u!w*I@0l327|rL^RIt7Kj_DzhCHVftr_#TT`5DO;Ijw zPg35Vq_QQ+cPq+2%|{fLq_=2DGRzvtc7!xn-He{!IOqdR)EGqckr&ai?p*XnLezcB zKl{E#fdT~z6ev)jK!E}U3KS?%pg@5F1qu`>P@q780tE^bC{Un4fr7V$-v9zrv*G|y F0082_7N-CJ literal 0 HcmV?d00001