mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-10-21 07:42:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Implement --find-links as flat indexes (directories in pip-compile) (#912)

Browse source 
Add directory `--find-links` support for local paths to pip-compile.

It seems that pip joins all sources and then picks the best package. We
explicitly give find links packages precedence if the same exists on an
index and locally by prefilling the `VersionMap`, otherwise they are
added as another index and the existing rules of precedence apply.

Internally, the feature is called _flat index_, which is more meaningful
than _find links_: We're not looking for links, we're picking up local
directories, and (TBD) support another index format that's just a flat
list of files instead of a nested index.

`RegistryBuiltDist` and `RegistrySourceDist` now use `WheelFilename` and
`SourceDistFilename` respectively. The `File` inside `RegistryBuiltDist`
and `RegistrySourceDist` gained the ability to represent both a url and
a path so that `--find-links` with a url and with a path works the same,
both being locked as `<package_name>@<version>` instead of
`<package_name> @ <url>`. (This is more of a detail, this PR in general
still work if we strip that and have directory find links represented as
`<package_name> @ file:///path/to/file.ext`)

`PrioritizedDistribution` and `FlatIndex` have been moved to locations
where we can use them in the upstack PR.

I added a `scripts/wheels` directory with stripped down wheels to use
for testing.

We're lacking tests for correct tag priority precedence with flat
indexes, i only confirmed this manually since it is not covered in the
pip-compile or pip-sync output.

Closes #876
This commit is contained in:
konsti 2024-01-15 03:04:10 +01:00 • committed by GitHub
parent 5ffbfadf66
commit e9b6b6fa36
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 1069 additions and 483 deletions
Download patch file Download diff file Expand all files Collapse all files

3
crates/distribution-types/Cargo.toml
View file

@ -14,9 +14,10 @@ workspace = true
[dependencies]
cache-key = { path = "../cache-key" }
distribution-filename = { path = "../distribution-filename" }
distribution-filename = { path = "../distribution-filename", features = ["serde"] }
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
platform-tags = { path = "../platform-tags" }
puffin-git = { path = "../puffin-git" }
puffin-normalize = { path = "../puffin-normalize" }
pypi-types = { path = "../pypi-types" }

31
crates/distribution-types/src/file.rs
View file

@ -1,9 +1,13 @@
use std::fmt::{Display, Formatter};
use std::path::PathBuf;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use url::Url;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use pep508_rs::VerbatimUrl;
use pypi_types::{BaseUrl, DistInfoMetadata, Hashes, Yanked};
/// Error converting [`pypi_types::File`] to [`distribution_type::File`].
@ -24,7 +28,7 @@ pub struct File {
pub requires_python: Option<VersionSpecifiers>,
pub size: Option<u64>,
pub upload_time: Option<DateTime<Utc>>,
pub url: Url,
pub url: FileLocation,
pub yanked: Option<Yanked>,
}
@ -38,10 +42,29 @@ impl File {
requires_python: file.requires_python.transpose()?,
size: file.size,
upload_time: file.upload_time,
url: base
.join_relative(&file.url)
.map_err(|err| FileConversionError::Url(file.url.clone(), err))?,
url: FileLocation::Url(
base.join_relative(&file.url)
.map_err(|err| FileConversionError::Url(file.url.clone(), err))?,
),
yanked: file.yanked,
})
}
}
/// While a registry file is generally a remote URL, it can also be a file if it comes from a directory flat indexes.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FileLocation {
/// URL relative to base
Url(Url),
/// Absolute path to file
Path(PathBuf, VerbatimUrl),
}
impl Display for FileLocation {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
FileLocation::Url(url) => Display::fmt(url, f),
FileLocation::Path(path, _url) => Display::fmt(&path.display(), f),
}
}
}

78
crates/distribution-types/src/index_url.rs
View file

@ -1,5 +1,5 @@
use std::iter::Chain;
use std::ops::Deref;
use std::path::PathBuf;
use std::str::FromStr;
use once_cell::sync::Lazy;
@ -53,6 +53,44 @@ impl Deref for IndexUrl {
}
}
/// A directory with distributions or a URL to an HTML file with a flat listing of distributions.
///
/// Also known as `--find-links`.
#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)]
pub enum FlatIndexLocation {
Path(PathBuf),
Url(Url),
}
impl FromStr for FlatIndexLocation {
type Err = FlatIndexError;
fn from_str(location: &str) -> Result<Self, Self::Err> {
if location.contains("://") {
let url =
Url::parse(location).map_err(|err| FlatIndexError::Url(location.into(), err))?;
if url.scheme() == "file" {
match url.to_file_path() {
Ok(path_buf) => Ok(Self::Path(path_buf)),
Err(()) => Err(FlatIndexError::FilePath(url)),
}
} else {
Ok(Self::Url(url))
}
} else {
Ok(Self::Path(PathBuf::from(location)))
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum FlatIndexError {
#[error("Invalid file location URL: {0}")]
Url(String, #[source] url::ParseError),
#[error("Invalid `file://` path in URL: {0}")]
FilePath(Url),
}
/// The index URLs to use for fetching packages.
///
/// "pip treats all package sources equally" (<https://github.com/pypa/pip/issues/8606#issuecomment-788754817>),
@ -60,34 +98,45 @@ impl Deref for IndexUrl {
///
/// If the fields are none and empty, ignore the package index, instead rely on local archives and
/// caches.
///
/// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`.
#[derive(Debug, Clone)]
pub struct IndexUrls {
pub index: Option<IndexUrl>,
pub extra_index: Vec<IndexUrl>,
pub struct IndexLocations {
index: Option<IndexUrl>,
extra_index: Vec<IndexUrl>,
flat_index: Vec<FlatIndexLocation>,
}
impl Default for IndexUrls {
impl Default for IndexLocations {
/// Just pypi
fn default() -> Self {
Self {
index: Some(IndexUrl::Pypi),
extra_index: Vec::new(),
flat_index: Vec::new(),
}
}
}
impl IndexUrls {
impl IndexLocations {
/// Determine the index URLs to use for fetching packages.
pub fn from_args(index: IndexUrl, extra_index: Vec<IndexUrl>, no_index: bool) -> Self {
pub fn from_args(
index: IndexUrl,
extra_index: Vec<IndexUrl>,
flat_index: Vec<FlatIndexLocation>,
no_index: bool,
) -> Self {
if no_index {
Self {
index: None,
extra_index: Vec::new(),
flat_index: Vec::new(),
}
} else {
Self {
index: Some(index),
extra_index,
flat_index,
}
}
}
@ -97,19 +146,12 @@ impl IndexUrls {
}
}
impl<'a> IntoIterator for &'a IndexUrls {
type Item = &'a IndexUrl;
type IntoIter = Chain<std::option::Iter<'a, IndexUrl>, std::slice::Iter<'a, IndexUrl>>;
fn into_iter(self) -> Self::IntoIter {
impl<'a> IndexLocations {
pub fn indexes(&'a self) -> impl Iterator<Item = &'a IndexUrl> + 'a {
self.index.iter().chain(self.extra_index.iter())
}
}
impl<'a> IndexUrls {
pub fn iter(
&'a self,
) -> Chain<std::option::Iter<'a, IndexUrl>, std::slice::Iter<'a, IndexUrl>> {
self.into_iter()
pub fn flat_indexes(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
self.flat_index.iter()
}
}

94
crates/distribution-types/src/lib.rs
View file

@ -25,8 +25,6 @@
//! * [`CachedRegistryDist`]
//! * [`CachedDirectUrlDist`]
//!
//! TODO(konstin): Track all kinds from [`Dist`].
//!
//! ## `InstalledDist`
//! An [`InstalledDist`] is built distribution (wheel) that is installed in a virtual environment,
//! with the two possible origins we currently track:
@ -34,8 +32,6 @@
//! * [`InstalledDirectUrlDist`]
//!
//! Since we read this information from [`direct_url.json`](https://packaging.python.org/en/latest/specifications/direct-url-data-structure/), it doesn't match the information [`Dist`] exactly.
//!
//! TODO(konstin): Track all kinds from [`Dist`].
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::str::FromStr;
@ -43,7 +39,7 @@ use std::str::FromStr;
use anyhow::Result;
use url::Url;
use distribution_filename::WheelFilename;
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
use pep440_rs::Version;
use pep508_rs::VerbatimUrl;
use puffin_normalize::PackageName;
@ -58,6 +54,7 @@ pub use crate::file::*;
pub use crate::id::*;
pub use crate::index_url::*;
pub use crate::installed::*;
pub use crate::prioritized_distribution::*;
pub use crate::resolution::*;
pub use crate::traits::*;
@ -70,6 +67,7 @@ mod file;
mod id;
mod index_url;
mod installed;
mod prioritized_distribution;
mod resolution;
mod traits;
@ -148,8 +146,7 @@ pub enum SourceDist {
/// A built distribution (wheel) that exists in a registry, like `PyPI`.
#[derive(Debug, Clone)]
pub struct RegistryBuiltDist {
pub name: PackageName,
pub version: Version,
pub filename: WheelFilename,
pub file: File,
pub index: IndexUrl,
}
@ -174,8 +171,7 @@ pub struct PathBuiltDist {
/// A source distribution that exists in a registry, like `PyPI`.
#[derive(Debug, Clone)]
pub struct RegistrySourceDist {
pub name: PackageName,
pub version: Version,
pub filename: SourceDistFilename,
pub file: File,
pub index: IndexUrl,
}
@ -207,24 +203,22 @@ pub struct PathSourceDist {
impl Dist {
/// Create a [`Dist`] for a registry-based distribution.
pub fn from_registry(name: PackageName, version: Version, file: File, index: IndexUrl) -> Self {
if Path::new(&file.filename)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
{
Self::Built(BuiltDist::Registry(RegistryBuiltDist {
name,
version,
file,
index,
}))
} else {
Self::Source(SourceDist::Registry(RegistrySourceDist {
name,
version,
file,
index,
}))
pub fn from_registry(filename: DistFilename, file: File, index: IndexUrl) -> Self {
match filename {
DistFilename::WheelFilename(filename) => {
Self::Built(BuiltDist::Registry(RegistryBuiltDist {
filename,
file,
index,
}))
}
DistFilename::SourceDistFilename(filename) => {
Self::Source(SourceDist::Registry(RegistrySourceDist {
filename,
file,
index,
}))
}
}
}
@ -305,6 +299,13 @@ impl Dist {
Dist::Source(source) => source.file(),
}
}
pub fn version(&self) -> Option<&Version> {
match self {
Dist::Built(wheel) => Some(wheel.version()),
Dist::Source(source_dist) => source_dist.version(),
}
}
}
impl BuiltDist {
@ -315,6 +316,14 @@ impl BuiltDist {
BuiltDist::DirectUrl(_) | BuiltDist::Path(_) => None,
}
}
pub fn version(&self) -> &Version {
match self {
BuiltDist::Registry(wheel) => &wheel.filename.version,
BuiltDist::DirectUrl(wheel) => &wheel.filename.version,
BuiltDist::Path(wheel) => &wheel.filename.version,
}
}
}
impl SourceDist {
@ -326,6 +335,13 @@ impl SourceDist {
}
}
pub fn version(&self) -> Option<&Version> {
match self {
SourceDist::Registry(source_dist) => Some(&source_dist.filename.version),
SourceDist::DirectUrl(_) | SourceDist::Git(_) | SourceDist::Path(_) => None,
}
}
#[must_use]
pub fn with_url(self, url: Url) -> Self {
match self {
@ -348,7 +364,7 @@ impl SourceDist {
impl Name for RegistryBuiltDist {
fn name(&self) -> &PackageName {
&self.name
&self.filename.name
}
}
@ -366,7 +382,7 @@ impl Name for PathBuiltDist {
impl Name for RegistrySourceDist {
fn name(&self) -> &PackageName {
&self.name
&self.filename.name
}
}
@ -420,7 +436,7 @@ impl Name for Dist {
impl DistributionMetadata for RegistryBuiltDist {
fn version_or_url(&self) -> VersionOrUrl {
VersionOrUrl::Version(&self.version)
VersionOrUrl::Version(&self.filename.version)
}
}
@ -438,7 +454,7 @@ impl DistributionMetadata for PathBuiltDist {
impl DistributionMetadata for RegistrySourceDist {
fn version_or_url(&self) -> VersionOrUrl {
VersionOrUrl::Version(&self.version)
VersionOrUrl::Version(&self.filename.version)
}
}
@ -678,6 +694,22 @@ impl Identifier for Path {
}
}
impl Identifier for FileLocation {
fn distribution_id(&self) -> DistributionId {
match self {
FileLocation::Url(url) => url.distribution_id(),
FileLocation::Path(path, _) => path.distribution_id(),
}
}
fn resource_id(&self) -> ResourceId {
match self {
FileLocation::Url(url) => url.resource_id(),
FileLocation::Path(path, _) => path.resource_id(),
}
}
}
impl Identifier for RegistryBuiltDist {
fn distribution_id(&self) -> DistributionId {
self.file.distribution_id()

207
crates/distribution-types/src/prioritized_distribution.rs Normal file
View file

@ -0,0 +1,207 @@
use pep440_rs::VersionSpecifiers;
use platform_tags::TagPriority;
use pypi_types::Hashes;
use crate::Dist;
/// Attach its requires-python to a [`Dist`], since downstream needs this information to filter
/// [`PrioritizedDistribution`].
#[derive(Debug, Clone)]
pub struct DistRequiresPython {
pub dist: Dist,
pub requires_python: Option<VersionSpecifiers>,
}
#[derive(Debug, Clone)]
pub struct PrioritizedDistribution {
/// An arbitrary source distribution for the package version.
source: Option<DistRequiresPython>,
/// The highest-priority, platform-compatible wheel for the package version.
compatible_wheel: Option<(DistRequiresPython, TagPriority)>,
/// An arbitrary, platform-incompatible wheel for the package version.
incompatible_wheel: Option<DistRequiresPython>,
/// The hashes for each distribution.
hashes: Vec<Hashes>,
}
impl PrioritizedDistribution {
/// Create a new [`PrioritizedDistribution`] from the given wheel distribution.
pub fn from_built(
dist: Dist,
requires_python: Option<VersionSpecifiers>,
hash: Option<Hashes>,
priority: Option<TagPriority>,
) -> Self {
if let Some(priority) = priority {
Self {
source: None,
compatible_wheel: Some((
DistRequiresPython {
dist,
requires_python,
},
priority,
)),
incompatible_wheel: None,
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(),
}
} else {
Self {
source: None,
compatible_wheel: None,
incompatible_wheel: Some(DistRequiresPython {
dist,
requires_python,
}),
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(),
}
}
}
/// Create a new [`PrioritizedDistribution`] from the given source distribution.
pub fn from_source(
dist: Dist,
requires_python: Option<VersionSpecifiers>,
hash: Option<Hashes>,
) -> Self {
Self {
source: Some(DistRequiresPython {
dist,
requires_python,
}),
compatible_wheel: None,
incompatible_wheel: None,
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(),
}
}
/// Insert the given built distribution into the [`PrioritizedDistribution`].
pub fn insert_built(
&mut self,
dist: Dist,
requires_python: Option<VersionSpecifiers>,
hash: Option<Hashes>,
priority: Option<TagPriority>,
) {
// Prefer the highest-priority, platform-compatible wheel.
if let Some(priority) = priority {
if let Some((.., existing_priority)) = &self.compatible_wheel {
if priority > *existing_priority {
self.compatible_wheel = Some((
DistRequiresPython {
dist,
requires_python,
},
priority,
));
}
} else {
self.compatible_wheel = Some((
DistRequiresPython {
dist,
requires_python,
},
priority,
));
}
} else if self.incompatible_wheel.is_none() {
self.incompatible_wheel = Some(DistRequiresPython {
dist,
requires_python,
});
}
if let Some(hash) = hash {
self.hashes.push(hash);
}
}
/// Insert the given source distribution into the [`PrioritizedDistribution`].
pub fn insert_source(
&mut self,
dist: Dist,
requires_python: Option<VersionSpecifiers>,
hash: Option<Hashes>,
) {
if self.source.is_none() {
self.source = Some(DistRequiresPython {
dist,
requires_python,
});
}
if let Some(hash) = hash {
self.hashes.push(hash);
}
}
/// Return the highest-priority distribution for the package version, if any.
pub fn get(&self) -> Option<ResolvableDist> {
match (
&self.compatible_wheel,
&self.source,
&self.incompatible_wheel,
) {
// Prefer the highest-priority, platform-compatible wheel.
(Some((wheel, tag_priority)), _, _) => {
Some(ResolvableDist::CompatibleWheel(wheel, *tag_priority))
}
// If we have a compatible source distribution and an incompatible wheel, return the
// wheel. We assume that all distributions have the same metadata for a given package
// version. If a compatible source distribution exists, we assume we can build it, but
// using the wheel is faster.
(_, Some(source_dist), Some(wheel)) => {
Some(ResolvableDist::IncompatibleWheel { source_dist, wheel })
}
// Otherwise, if we have a source distribution, return it.
(_, Some(source_dist), _) => Some(ResolvableDist::SourceDist(source_dist)),
_ => None,
}
}
/// Return the hashes for each distribution.
pub fn hashes(&self) -> &[Hashes] {
&self.hashes
}
}
#[derive(Debug, Clone)]
pub enum ResolvableDist<'a> {
/// The distribution should be resolved and installed using a source distribution.
SourceDist(&'a DistRequiresPython),
/// The distribution should be resolved and installed using a wheel distribution.
CompatibleWheel(&'a DistRequiresPython, TagPriority),
/// The distribution should be resolved using an incompatible wheel distribution, but
/// installed using a source distribution.
IncompatibleWheel {
source_dist: &'a DistRequiresPython,
wheel: &'a DistRequiresPython,
},
}
impl<'a> ResolvableDist<'a> {
/// Return the [`DistRequiresPython`] to use during resolution.
pub fn resolve(&self) -> &DistRequiresPython {
match *self {
ResolvableDist::SourceDist(sdist) => sdist,
ResolvableDist::CompatibleWheel(wheel, _) => wheel,
ResolvableDist::IncompatibleWheel {
source_dist: _,
wheel,
} => wheel,
}
}
/// Return the [`DistRequiresPython`] to use during installation.
pub fn install(&self) -> &DistRequiresPython {
match *self {
ResolvableDist::SourceDist(sdist) => sdist,
ResolvableDist::CompatibleWheel(wheel, _) => wheel,
ResolvableDist::IncompatibleWheel {
source_dist,
wheel: _,
} => source_dist,
}
}
}

8
crates/distribution-types/src/resolution.rs
View file

@ -65,11 +65,11 @@ impl From<Dist> for Requirement {
fn from(dist: Dist) -> Self {
match dist {
Dist::Built(BuiltDist::Registry(wheel)) => Requirement {
name: wheel.name,
name: wheel.filename.name,
extras: None,
version_or_url: Some(pep508_rs::VersionOrUrl::VersionSpecifier(
pep440_rs::VersionSpecifiers::from(
pep440_rs::VersionSpecifier::equals_version(wheel.version),
pep440_rs::VersionSpecifier::equals_version(wheel.filename.version),
),
)),
marker: None,
@ -87,11 +87,11 @@ impl From<Dist> for Requirement {
marker: None,
},
Dist::Source(SourceDist::Registry(sdist)) => Requirement {
name: sdist.name,
name: sdist.filename.name,
extras: None,
version_or_url: Some(pep508_rs::VersionOrUrl::VersionSpecifier(
pep440_rs::VersionSpecifiers::from(
pep440_rs::VersionSpecifier::equals_version(sdist.version),
pep440_rs::VersionSpecifier::equals_version(sdist.filename.version),
),
)),
marker: None,