From ae659c8bfed9666d2f3152b39aceb0c41f834fda Mon Sep 17 00:00:00 2001 From: konsti Date: Fri, 20 Dec 2024 10:28:46 +0100 Subject: [PATCH] Stable order for virtual packages (#10024) uv gives priorities to packages by package name, not by virtual package (`PubGrubPackage`). pubgrub otoh when prioritizing order the virtual packages. When the order of virtual packages changes, uv changes its resolutions and error messages. This means uv was depending on implementation details of pubgrub's prioritization caching. This broke with https://github.com/pubgrub-rs/pubgrub/pull/299, which added a tiebreaker term that made pubgrub's sorting deterministic given a deterministic ordering of allocating the packages (which happens the first time pubgrub sees a package). The new custom tiebreaker decreases the difference to upstream pubgrub. --- Cargo.lock | 4 +- Cargo.toml | 4 +- crates/uv-resolver/src/dependency_provider.rs | 3 +- crates/uv-resolver/src/pubgrub/priority.rs | 45 +++-- crates/uv/tests/it/lock_conflict.rs | 16 +- crates/uv/tests/it/pip_install_scenarios.rs | 14 +- ...it__ecosystem__transformers-lock-file.snap | 154 ++++++++++++++---- ...cosystem__transformers-uv-lock-output.snap | 2 +- 8 files changed, 179 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 357365c4b..5761153f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2673,7 +2673,7 @@ dependencies = [ [[package]] name = "pubgrub" version = "0.2.1" -source = "git+https://github.com/astral-sh/pubgrub?rev=05e8d12cea8d72c6d2d017900e478d0abd28fef4#05e8d12cea8d72c6d2d017900e478d0abd28fef4" +source = "git+https://github.com/astral-sh/pubgrub?rev=648aa343486e5529953153781fc86025c73c4a61#648aa343486e5529953153781fc86025c73c4a61" dependencies = [ "indexmap", "log", @@ -5676,7 +5676,7 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version-ranges" version = "0.1.1" -source = "git+https://github.com/astral-sh/pubgrub?rev=05e8d12cea8d72c6d2d017900e478d0abd28fef4#05e8d12cea8d72c6d2d017900e478d0abd28fef4" +source = "git+https://github.com/astral-sh/pubgrub?rev=648aa343486e5529953153781fc86025c73c4a61#648aa343486e5529953153781fc86025c73c4a61" dependencies = [ "smallvec", ] diff --git a/Cargo.toml b/Cargo.toml index 634ce793e..197d7ab11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,7 +130,7 @@ petgraph = { version = "0.6.5" } platform-info = { version = "2.0.3" } proc-macro2 = { version = "1.0.86" } procfs = { version = "0.17.0", default-features = false, features = ["flate2"] } -pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "05e8d12cea8d72c6d2d017900e478d0abd28fef4" } +pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "648aa343486e5529953153781fc86025c73c4a61" } quote = { version = "1.0.37" } rayon = { version = "1.10.0" } reflink-copy = { version = "0.1.19" } @@ -175,7 +175,7 @@ unicode-width = { version = "0.1.13" } unscanny = { version = "0.1.0" } url = { version = "2.5.2", features = ["serde"] } urlencoding = { version = "2.1.3" } -version-ranges = { git = "https://github.com/astral-sh/pubgrub", rev = "05e8d12cea8d72c6d2d017900e478d0abd28fef4" } +version-ranges = { git = "https://github.com/astral-sh/pubgrub", rev = "648aa343486e5529953153781fc86025c73c4a61" } walkdir = { version = "2.5.0" } which = { version = "7.0.0", features = ["regex"] } windows-registry = { version = "0.3.0" } diff --git a/crates/uv-resolver/src/dependency_provider.rs b/crates/uv-resolver/src/dependency_provider.rs index 66ce64d1f..d2e1792fe 100644 --- a/crates/uv-resolver/src/dependency_provider.rs +++ b/crates/uv-resolver/src/dependency_provider.rs @@ -17,7 +17,8 @@ impl DependencyProvider for UvDependencyProvider { type V = Version; type VS = Range; type M = UnavailableReason; - type Priority = Option; + /// Main priority and tiebreak for virtual packages + type Priority = (Option, u32); type Err = Infallible; fn prioritize( diff --git a/crates/uv-resolver/src/pubgrub/priority.rs b/crates/uv-resolver/src/pubgrub/priority.rs index e11aba050..3fbd4b2a9 100644 --- a/crates/uv-resolver/src/pubgrub/priority.rs +++ b/crates/uv-resolver/src/pubgrub/priority.rs @@ -20,7 +20,10 @@ use crate::pubgrub::PubGrubPackageInner; /// /// See: #[derive(Clone, Debug, Default)] -pub(crate) struct PubGrubPriorities(FxHashMap); +pub(crate) struct PubGrubPriorities { + package_priority: FxHashMap, + virtual_package_tiebreaker: FxHashMap, +} impl PubGrubPriorities { /// Add a [`PubGrubPackage`] to the priority map. @@ -30,14 +33,22 @@ impl PubGrubPriorities { version: &Range, urls: &ForkUrls, ) { - let next = self.0.len(); + if !self.virtual_package_tiebreaker.contains_key(package) { + self.virtual_package_tiebreaker.insert( + package.clone(), + u32::try_from(self.virtual_package_tiebreaker.len()) + .expect("Less than 2**32 packages"), + ); + } + + let next = self.package_priority.len(); // The root package and Python constraints have no explicit priority, the root package is // always first and the Python version (range) is fixed. let Some(name) = package.name_no_root() else { return; }; - match self.0.entry(name.clone()) { + match self.package_priority.entry(name.clone()) { std::collections::hash_map::Entry::Occupied(mut entry) => { // Preserve the original index. let index = Self::get_index(&entry).unwrap_or(next); @@ -92,15 +103,21 @@ impl PubGrubPriorities { } /// Return the [`PubGrubPriority`] of the given package, if it exists. - pub(crate) fn get(&self, package: &PubGrubPackage) -> Option { - match &**package { + pub(crate) fn get(&self, package: &PubGrubPackage) -> (Option, u32) { + let package_priority = match &**package { PubGrubPackageInner::Root(_) => Some(PubGrubPriority::Root), PubGrubPackageInner::Python(_) => Some(PubGrubPriority::Root), - PubGrubPackageInner::Marker { name, .. } => self.0.get(name).copied(), - PubGrubPackageInner::Extra { name, .. } => self.0.get(name).copied(), - PubGrubPackageInner::Dev { name, .. } => self.0.get(name).copied(), - PubGrubPackageInner::Package { name, .. } => self.0.get(name).copied(), - } + PubGrubPackageInner::Marker { name, .. } => self.package_priority.get(name).copied(), + PubGrubPackageInner::Extra { name, .. } => self.package_priority.get(name).copied(), + PubGrubPackageInner::Dev { name, .. } => self.package_priority.get(name).copied(), + PubGrubPackageInner::Package { name, .. } => self.package_priority.get(name).copied(), + }; + let virtual_package_tiebreaker = self + .virtual_package_tiebreaker + .get(package) + .copied() + .unwrap_or_default(); + (package_priority, virtual_package_tiebreaker) } /// Mark a package as prioritized by setting it to [`PubGrubPriority::ConflictEarly`], if it @@ -109,7 +126,7 @@ impl PubGrubPriorities { /// Returns whether the priority was changed, i.e., it's the first time we hit this condition /// for the package. pub(crate) fn mark_conflict_early(&mut self, package: &PubGrubPackage) -> bool { - let next = self.0.len(); + let next = self.package_priority.len(); let Some(name) = package.name_no_root() else { // Not a correctness bug if cfg!(debug_assertions) { @@ -118,7 +135,7 @@ impl PubGrubPriorities { return false; } }; - match self.0.entry(name.clone()) { + match self.package_priority.entry(name.clone()) { std::collections::hash_map::Entry::Occupied(mut entry) => { if matches!( entry.get(), @@ -144,7 +161,7 @@ impl PubGrubPriorities { /// Returns whether the priority was changed, i.e., it's the first time this package was /// marked as conflicting above the threshold. pub(crate) fn mark_conflict_late(&mut self, package: &PubGrubPackage) -> bool { - let next = self.0.len(); + let next = self.package_priority.len(); let Some(name) = package.name_no_root() else { // Not a correctness bug if cfg!(debug_assertions) { @@ -153,7 +170,7 @@ impl PubGrubPriorities { return false; } }; - match self.0.entry(name.clone()) { + match self.package_priority.entry(name.clone()) { std::collections::hash_map::Entry::Occupied(mut entry) => { // The ConflictEarly` match avoids infinite loops. if matches!( diff --git a/crates/uv/tests/it/lock_conflict.rs b/crates/uv/tests/it/lock_conflict.rs index 56ea22c5a..a056f7b36 100644 --- a/crates/uv/tests/it/lock_conflict.rs +++ b/crates/uv/tests/it/lock_conflict.rs @@ -250,8 +250,8 @@ fn extra_basic_three_extras() -> Result<()> { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because project[project3] depends on sortedcontainers==2.4.0 and project[extra1] depends on sortedcontainers==2.2.0, we can conclude that project[extra1] and project[project3] are incompatible. - And because your project requires project[extra1] and project[project3], we can conclude that your project's requirements are unsatisfiable. + ╰─▶ Because project[project3] depends on sortedcontainers==2.4.0 and project[extra2] depends on sortedcontainers==2.3.0, we can conclude that project[extra2] and project[project3] are incompatible. + And because your project requires project[extra2] and project[project3], we can conclude that your project's requirements are unsatisfiable. "###); // And now with the same extra configuration, we tell uv about @@ -538,8 +538,8 @@ fn extra_multiple_not_conflicting2() -> Result<()> { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because project[project4] depends on sortedcontainers==2.4.0 and project[extra1] depends on sortedcontainers==2.3.0, we can conclude that project[extra1] and project[project4] are incompatible. - And because your project requires project[extra1] and project[project4], we can conclude that your project's requirements are unsatisfiable. + ╰─▶ Because project[project4] depends on sortedcontainers==2.4.0 and project[project3] depends on sortedcontainers==2.3.0, we can conclude that project[project3] and project[project4] are incompatible. + And because your project requires project[project3] and project[project4], we can conclude that your project's requirements are unsatisfiable. "###); // If we define extra1/extra2 as conflicting and project3/project4 @@ -1289,10 +1289,8 @@ fn extra_nested_across_workspace() -> Result<()> { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because dummy[extra2] depends on proxy1[extra2] and only proxy1[extra2]==0.1.0 is available, we can conclude that dummy[extra2] depends on proxy1[extra2]==0.1.0. (1) - - Because proxy1[extra2]==0.1.0 depends on anyio==4.2.0 and proxy1[extra1]==0.1.0 depends on anyio==4.1.0, we can conclude that proxy1[extra1]==0.1.0 and proxy1[extra2]==0.1.0 are incompatible. - And because we know from (1) that dummy[extra2] depends on proxy1[extra2]==0.1.0, we can conclude that dummy[extra2] and proxy1[extra1]==0.1.0 are incompatible. + ╰─▶ Because dummy[extra2] depends on proxy1[extra2] and only proxy1[extra2]==0.1.0 is available, we can conclude that dummy[extra2] depends on proxy1[extra2]==0.1.0. + And because proxy1[extra2]==0.1.0 depends on anyio==4.2.0 and proxy1[extra1]==0.1.0 depends on anyio==4.1.0, we can conclude that proxy1[extra1]==0.1.0 and dummy[extra2] are incompatible. And because only proxy1[extra1]==0.1.0 is available and dummysub[extra1] depends on proxy1[extra1], we can conclude that dummysub[extra1] and dummy[extra2] are incompatible. And because your workspace requires dummy[extra2] and dummysub[extra1], we can conclude that your workspace's requirements are unsatisfiable. "###); @@ -1795,7 +1793,7 @@ fn mixed() -> Result<()> { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because project[extra1] depends on sortedcontainers==2.4.0 and project:group1 depends on sortedcontainers==2.3.0, we can conclude that project:group1 and project[extra1] are incompatible. + ╰─▶ Because project:group1 depends on sortedcontainers==2.3.0 and project[extra1] depends on sortedcontainers==2.4.0, we can conclude that project[extra1] and project:group1 are incompatible. And because your project requires project[extra1] and project:group1, we can conclude that your project's requirements are unsatisfiable. "###); diff --git a/crates/uv/tests/it/pip_install_scenarios.rs b/crates/uv/tests/it/pip_install_scenarios.rs index 2ab7b765c..cd8882c87 100644 --- a/crates/uv/tests/it/pip_install_scenarios.rs +++ b/crates/uv/tests/it/pip_install_scenarios.rs @@ -334,10 +334,11 @@ fn dependency_excludes_non_contiguous_range_of_compatible_versions() { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because only the following versions of package-a are available: + ╰─▶ Because package-a==1.0.0 depends on package-b==1.0.0 and only the following versions of package-a are available: package-a==1.0.0 - package-a>2.0.0,<=3.0.0 - and package-a==1.0.0 depends on package-b==1.0.0, we can conclude that package-a<2.0.0 depends on package-b==1.0.0. (1) + package-a>2.0.0 + we can conclude that package-a<2.0.0 depends on package-b==1.0.0. + And because only package-a<=3.0.0 is available, we can conclude that package-a<2.0.0 depends on package-b==1.0.0. (1) Because only the following versions of package-c are available: package-c==1.0.0 @@ -445,10 +446,11 @@ fn dependency_excludes_range_of_compatible_versions() { ----- stderr ----- × No solution found when resolving dependencies: - ╰─▶ Because only the following versions of package-a are available: + ╰─▶ Because package-a==1.0.0 depends on package-b==1.0.0 and only the following versions of package-a are available: package-a==1.0.0 - package-a>2.0.0,<=3.0.0 - and package-a==1.0.0 depends on package-b==1.0.0, we can conclude that package-a<2.0.0 depends on package-b==1.0.0. (1) + package-a>2.0.0 + we can conclude that package-a<2.0.0 depends on package-b==1.0.0. + And because only package-a<=3.0.0 is available, we can conclude that package-a<2.0.0 depends on package-b==1.0.0. (1) Because only the following versions of package-c are available: package-c==1.0.0 diff --git a/crates/uv/tests/it/snapshots/it__ecosystem__transformers-lock-file.snap b/crates/uv/tests/it/snapshots/it__ecosystem__transformers-lock-file.snap index eb785d0fb..67d4e4fe2 100644 --- a/crates/uv/tests/it/snapshots/it__ecosystem__transformers-lock-file.snap +++ b/crates/uv/tests/it/snapshots/it__ecosystem__transformers-lock-file.snap @@ -774,26 +774,71 @@ wheels = [ name = "datasets" version = "2.14.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.10.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.10.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.10' and platform_machine == 'arm64' and sys_platform == 'darwin'", + "python_full_version < '3.10' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.10' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.10' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.10' and sys_platform != 'darwin' and sys_platform != 'linux')", +] dependencies = [ - { name = "aiohttp" }, - { name = "dill" }, - { name = "fsspec", extra = ["http"] }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, + { name = "aiohttp", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "dill", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, extra = ["http"], marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "huggingface-hub", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "multiprocess", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "packaging", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "pandas", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "pyarrow", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "requests", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "tqdm", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "xxhash", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/69/8cc725b5d38968fd118e4ce56a483b16e75b7793854c1a392ec4a34eeb31/datasets-2.14.4.tar.gz", hash = "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b", size = 2178719 } wheels = [ { url = "https://files.pythonhosted.org/packages/66/f8/38298237d18d4b6a8ee5dfe390e97bed5adb8e01ec6f9680c0ddf3066728/datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b", size = 519335 }, ] +[[package]] +name = "datasets" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", +] +dependencies = [ + { name = "aiohttp", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "dill", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "filelock", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, extra = ["http"], marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "huggingface-hub", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "multiprocess", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "numpy", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "packaging", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "pandas", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "pyarrow", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "pyarrow-hotfix", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "pyyaml", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "requests", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "tqdm", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "xxhash", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/59/b94bfb5f6225c4c931cd516390b3f006e232a036a48337f72889c6c9ab27/datasets-2.20.0.tar.gz", hash = "sha256:3c4dbcd27e0f642b9d41d20ff2efa721a5e04b32b2ca4009e0fc9139e324553f", size = 2225757 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/2d/963b266bb8f88492d5ab4232d74292af8beb5b6fdae97902df9e284d4c32/datasets-2.20.0-py3-none-any.whl", hash = "sha256:76ac02e3bdfff824492e20678f0b6b1b6d080515957fe834b00c2ba8d6b18e5e", size = 547777 }, +] + [[package]] name = "decorator" version = "5.1.1" @@ -900,7 +945,7 @@ wheels = [ [package.optional-dependencies] epath = [ - { name = "fsspec", marker = "python_full_version < '3.10'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "importlib-resources", marker = "python_full_version < '3.10'" }, { name = "typing-extensions", marker = "python_full_version < '3.10'" }, { name = "zipp", marker = "python_full_version < '3.10'" }, @@ -925,7 +970,7 @@ wheels = [ [package.optional-dependencies] epath = [ - { name = "fsspec", marker = "python_full_version == '3.10.*'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "importlib-resources", marker = "python_full_version == '3.10.*'" }, { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, { name = "zipp", marker = "python_full_version == '3.10.*'" }, @@ -956,7 +1001,8 @@ wheels = [ [package.optional-dependencies] epath = [ - { name = "fsspec", marker = "python_full_version >= '3.11'" }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and python_full_version < '3.13') or (python_full_version >= '3.11' and sys_platform == 'darwin')" }, { name = "importlib-resources", marker = "python_full_version >= '3.11'" }, { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, { name = "zipp", marker = "python_full_version >= '3.11'" }, @@ -970,9 +1016,11 @@ name = "evaluate" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "dill" }, - { name = "fsspec", extra = ["http"] }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, extra = ["http"], marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, extra = ["http"], marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, { name = "huggingface-hub" }, { name = "multiprocess" }, { name = "numpy" }, @@ -1194,10 +1242,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/10/466fe96dae1bff622021ee687f68e5524d6392b0a2f80d05001cd3a451ba/frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7", size = 11552 }, ] +[[package]] +name = "fsspec" +version = "2024.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", +] +sdist = { url = "https://files.pythonhosted.org/packages/71/28/cbf337fddd6f22686b7c2639b80e006accd904db152fe333fd98f4cd8d1e/fsspec-2024.5.0.tar.gz", hash = "sha256:1d021b0b0f933e3b3029ed808eb400c08ba101ca2de4b3483fbc9ca23fcee94a", size = 400066 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/a3/16e9fe32187e9c8bc7f9b7bcd9728529faa725231a0c96f2f98714ff2fc5/fsspec-2024.5.0-py3-none-any.whl", hash = "sha256:e0fdbc446d67e182f49a70b82cf7889028a63588fde6b222521f10937b2b670c", size = 316106 }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp", marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, +] + [[package]] name = "fsspec" version = "2024.6.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.10.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.10.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.10' and platform_machine == 'arm64' and sys_platform == 'darwin'", + "python_full_version < '3.10' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.10' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.10' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.10' and sys_platform != 'darwin' and sys_platform != 'linux')", +] sdist = { url = "https://files.pythonhosted.org/packages/90/b6/eba5024a9889fcfff396db543a34bef0ab9d002278f163129f9f01005960/fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49", size = 284584 } wheels = [ { url = "https://files.pythonhosted.org/packages/5e/44/73bea497ac69bafde2ee4269292fa3b41f1198f4bb7bbaaabde30ad29d4a/fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e", size = 177561 }, @@ -1205,7 +1286,7 @@ wheels = [ [package.optional-dependencies] http = [ - { name = "aiohttp" }, + { name = "aiohttp", marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, ] [[package]] @@ -1484,7 +1565,8 @@ version = "0.24.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, - { name = "fsspec" }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, { name = "packaging" }, { name = "pyyaml" }, { name = "requests" }, @@ -3090,6 +3172,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/f6/b75d4816c32f1618ed31a005ee635dd1d91d8164495d94f2ea092f594661/pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204", size = 25148611 }, ] +[[package]] +name = "pyarrow-hotfix" +version = "0.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/0a/71da7b0db0c7078d4cf34ecf0c70ded5ed29decc06612097474e0114f4cc/pyarrow_hotfix-0.6.tar.gz", hash = "sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945", size = 9754 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/f4/9ec2222f5f5f8ea04f66f184caafd991a39c8782e31f5b0266f101cb68ca/pyarrow_hotfix-0.6-py3-none-any.whl", hash = "sha256:dcc9ae2d220dff0083be6a9aa8e0cdee5182ad358d4931fce825c545e5c89178", size = 7888 }, +] + [[package]] name = "pyasn1" version = "0.6.0" @@ -3481,7 +3572,8 @@ wheels = [ [package.optional-dependencies] tune = [ - { name = "fsspec" }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, { name = "pandas" }, { name = "pyarrow" }, { name = "requests" }, @@ -4841,7 +4933,8 @@ version = "2.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, - { name = "fsspec" }, + { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, + { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, { name = "jinja2" }, { name = "networkx", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "networkx", version = "3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -4981,7 +5074,8 @@ accelerate = [ ] agents = [ { name = "accelerate" }, - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "diffusers" }, { name = "opencv-python" }, { name = "pillow" }, @@ -5035,7 +5129,8 @@ deepspeed-testing = [ { name = "accelerate" }, { name = "beautifulsoup4" }, { name = "cookiecutter" }, - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "deepspeed" }, { name = "dill" }, { name = "evaluate" }, @@ -5066,7 +5161,8 @@ dev-dependencies = [ { name = "beautifulsoup4" }, { name = "codecarbon" }, { name = "cookiecutter" }, - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "decord" }, { name = "dill" }, { name = "evaluate" }, @@ -5204,7 +5300,8 @@ optuna = [ { name = "optuna" }, ] quality = [ - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "gitpython" }, { name = "hf-doc-builder" }, { name = "isort" }, @@ -5215,7 +5312,8 @@ ray = [ { name = "ray", extra = ["tune"] }, ] retrieval = [ - { name = "datasets" }, + { name = "datasets", version = "2.14.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or sys_platform == 'darwin'" }, + { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and sys_platform != 'darwin'" }, { name = "faiss-cpu" }, ] sagemaker = [ @@ -5579,7 +5677,7 @@ name = "triton" version = "3.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "python_full_version < '3.13'" }, + { name = "filelock" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/45/27/14cc3101409b9b4b9241d2ba7deaa93535a217a211c86c4cc7151fb12181/triton-3.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e1efef76935b2febc365bfadf74bcb65a6f959a9872e5bddf44cc9e0adce1e1a", size = 209376304 }, diff --git a/crates/uv/tests/it/snapshots/it__ecosystem__transformers-uv-lock-output.snap b/crates/uv/tests/it/snapshots/it__ecosystem__transformers-uv-lock-output.snap index 2a7888962..f1a57fbd8 100644 --- a/crates/uv/tests/it/snapshots/it__ecosystem__transformers-uv-lock-output.snap +++ b/crates/uv/tests/it/snapshots/it__ecosystem__transformers-uv-lock-output.snap @@ -7,4 +7,4 @@ exit_code: 0 ----- stdout ----- ----- stderr ----- -Resolved 285 packages in [TIME] +Resolved 288 packages in [TIME]