mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-11-01 20:31:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Read requirements from requires.txt when available (#6655)

Browse source 
## Summary

Allows us to avoid building setuptools-based packages at versions prior
to Metadata 2.2

Closes https://github.com/astral-sh/uv/issues/6647.
This commit is contained in:
Charlie Marsh 2024-08-27 09:02:26 -04:00 • committed by GitHub
parent 51723a2699
commit ce749591de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 362 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

166
crates/pypi-types/src/metadata.rs
View file

@ -1,5 +1,6 @@
//! Derived from `pypi_types_crate`.
use std::io::BufRead;
use std::str::FromStr;
use indexmap::IndexMap;
@ -10,7 +11,8 @@ use thiserror::Error;
use tracing::warn;
use pep440_rs::{Version, VersionParseError, VersionSpecifiers, VersionSpecifiersParseError};
use pep508_rs::{Pep508Error, Requirement};
use pep508_rs::marker::MarkerValueExtra;
use pep508_rs::{ExtraOperator, MarkerExpression, MarkerTree, Pep508Error, Requirement};
use uv_normalize::{ExtraName, InvalidNameError, PackageName};
use crate::lenient_requirement::LenientRequirement;
@ -62,6 +64,8 @@ pub enum MetadataError {
DynamicField(&'static str),
#[error("The project uses Poetry's syntax to declare its dependencies, despite including a `project` table in `pyproject.toml`")]
PoetrySyntax,
#[error("Failed to read `requires.txt` contents")]
RequiresTxtContents(#[from] std::io::Error),
}
impl From<Pep508Error<VerbatimParsedUrl>> for MetadataError {
@ -492,6 +496,109 @@ impl RequiresDist {
}
}
/// `requires.txt` metadata as defined in <https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#dependency-metadata>.
///
/// This is a subset of the full metadata specification, and only includes the fields that are
/// included in the legacy `requires.txt` file.
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct RequiresTxt {
pub requires_dist: Vec<Requirement<VerbatimParsedUrl>>,
pub provides_extras: Vec<ExtraName>,
}
impl RequiresTxt {
/// Parse the [`RequiresTxt`] from a `requires.txt` file, as included in an `egg-info`.
///
/// See: <https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#dependency-metadata>
pub fn parse(content: &[u8]) -> Result<Self, MetadataError> {
let mut requires_dist = vec![];
let mut provides_extras = vec![];
let mut current_marker = MarkerTree::default();
for line in content.lines() {
let line = line.map_err(MetadataError::RequiresTxtContents)?;
let line = line.trim();
if line.is_empty() {
continue;
}
// When encountering a new section, parse the extra and marker from the header, e.g.,
// `[:sys_platform == "win32"]` or `[dev]`.
if line.starts_with('[') {
let line = line.trim_start_matches('[').trim_end_matches(']');
// Split into extra and marker, both of which can be empty.
let (extra, marker) = {
let (extra, marker) = match line.split_once(':') {
Some((extra, marker)) => (Some(extra), Some(marker)),
None => (Some(line), None),
};
let extra = extra.filter(|extra| !extra.is_empty());
let marker = marker.filter(|marker| !marker.is_empty());
(extra, marker)
};
// Parse the extra.
let extra = if let Some(extra) = extra {
if let Ok(extra) = ExtraName::from_str(extra) {
provides_extras.push(extra.clone());
Some(MarkerValueExtra::Extra(extra))
} else {
Some(MarkerValueExtra::Arbitrary(extra.to_string()))
}
} else {
None
};
// Parse the marker.
let marker = marker.map(MarkerTree::parse_str).transpose()?;
// Create the marker tree.
match (extra, marker) {
(Some(extra), Some(mut marker)) => {
marker.and(MarkerTree::expression(MarkerExpression::Extra {
operator: ExtraOperator::Equal,
name: extra,
}));
current_marker = marker;
}
(Some(extra), None) => {
current_marker = MarkerTree::expression(MarkerExpression::Extra {
operator: ExtraOperator::Equal,
name: extra,
});
}
(None, Some(marker)) => {
current_marker = marker;
}
(None, None) => {
current_marker = MarkerTree::default();
}
}
continue;
}
// Parse the requirement.
let requirement =
Requirement::<VerbatimParsedUrl>::from(LenientRequirement::from_str(line)?);
// Add the markers and extra, if necessary.
requires_dist.push(Requirement {
marker: current_marker.clone(),
..requirement
});
}
Ok(Self {
requires_dist,
provides_extras,
})
}
}
/// The headers of a distribution metadata file.
#[derive(Debug)]
struct Headers<'a>(Vec<mailparse::MailHeader<'a>>);
@ -531,7 +638,7 @@ mod tests {
use pep440_rs::Version;
use uv_normalize::PackageName;
use crate::MetadataError;
use crate::{MetadataError, RequiresTxt};
use super::Metadata23;
@ -677,4 +784,59 @@ mod tests {
);
assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]);
}
#[test]
fn test_requires_txt() {
let s = r"
Werkzeug>=0.14
Jinja2>=2.10
[dev]
pytest>=3
sphinx
[dotenv]
python-dotenv
";
let meta = RequiresTxt::parse(s.as_bytes()).unwrap();
assert_eq!(
meta.requires_dist,
vec![
"Werkzeug>=0.14".parse().unwrap(),
"Jinja2>=2.10".parse().unwrap(),
"pytest>=3; extra == \"dev\"".parse().unwrap(),
"sphinx; extra == \"dev\"".parse().unwrap(),
"python-dotenv; extra == \"dotenv\"".parse().unwrap(),
]
);
let s = r"
Werkzeug>=0.14
[dev:]
Jinja2>=2.10
[:sys_platform == 'win32']
pytest>=3
[]
sphinx
[dotenv:sys_platform == 'darwin']
python-dotenv
";
let meta = RequiresTxt::parse(s.as_bytes()).unwrap();
assert_eq!(
meta.requires_dist,
vec![
"Werkzeug>=0.14".parse().unwrap(),
"Jinja2>=2.10 ; extra == \"dev\"".parse().unwrap(),
"pytest>=3; sys_platform == 'win32'".parse().unwrap(),
"sphinx".parse().unwrap(),
"python-dotenv; sys_platform == 'darwin' and extra == \"dotenv\""
.parse()
.unwrap(),
]
);
}
}

6
crates/uv-distribution/src/error.rs
View file

@ -71,8 +71,14 @@ pub enum Error {
Extract(#[from] uv_extract::Error),
#[error("The source distribution is missing a `PKG-INFO` file")]
MissingPkgInfo,
#[error("The source distribution is missing an `egg-info` directory")]
MissingEggInfo,
#[error("The source distribution is missing a `requires.txt` file")]
MissingRequiresTxt,
#[error("Failed to extract static metadata from `PKG-INFO`")]
PkgInfo(#[source] pypi_types::MetadataError),
#[error("Failed to extract metadata from `requires.txt`")]
RequiresTxt(#[source] pypi_types::MetadataError),
#[error("The source distribution is missing a `pyproject.toml` file")]
MissingPyprojectToml,
#[error("Failed to extract static metadata from `pyproject.toml`")]

181
crates/uv-distribution/src/source/mod.rs
View file

@ -20,7 +20,7 @@ use distribution_types::{
};
use install_wheel_rs::metadata::read_archive_metadata;
use platform_tags::Tags;
use pypi_types::{HashDigest, Metadata23};
use pypi_types::{HashDigest, Metadata12, Metadata23, RequiresTxt};
use uv_cache::{
ArchiveTimestamp, CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Timestamp, WheelCache,
};
@ -1505,31 +1505,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
source_root: &Path,
subdirectory: Option<&Path>,
) -> Result<Option<Metadata23>, Error> {
// Attempt to read static metadata from the `PKG-INFO` file.
match read_pkg_info(source_root, subdirectory).await {
Ok(metadata) => {
debug!("Found static `PKG-INFO` for: {source}");
// Validate the metadata.
validate(source, &metadata)?;
return Ok(Some(metadata));
}
Err(
err @ (Error::MissingPkgInfo
| Error::PkgInfo(
pypi_types::MetadataError::Pep508Error(_)
| pypi_types::MetadataError::DynamicField(_)
| pypi_types::MetadataError::FieldNotFound(_)
| pypi_types::MetadataError::UnsupportedMetadataVersion(_)
| pypi_types::MetadataError::PoetrySyntax,
)),
) => {
debug!("No static `PKG-INFO` available for: {source} ({err:?})");
}
Err(err) => return Err(err),
}
// Attempt to read static metadata from the `pyproject.toml`.
match read_pyproject_toml(source_root, subdirectory).await {
Ok(metadata) => {
@ -1546,7 +1521,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
pypi_types::MetadataError::Pep508Error(_)
| pypi_types::MetadataError::DynamicField(_)
| pypi_types::MetadataError::FieldNotFound(_)
| pypi_types::MetadataError::UnsupportedMetadataVersion(_)
| pypi_types::MetadataError::PoetrySyntax,
)),
) => {
@ -1555,6 +1529,60 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
Err(err) => return Err(err),
}
// Attempt to read static metadata from the `PKG-INFO` file.
match read_pkg_info(source_root, subdirectory).await {
Ok(metadata) => {
debug!("Found static `PKG-INFO` for: {source}");
// Validate the metadata.
validate(source, &metadata)?;
return Ok(Some(metadata));
}
Err(
err @ (Error::MissingPkgInfo
| Error::PkgInfo(
pypi_types::MetadataError::Pep508Error(_)
| pypi_types::MetadataError::DynamicField(_)
| pypi_types::MetadataError::FieldNotFound(_)
| pypi_types::MetadataError::UnsupportedMetadataVersion(_),
)),
) => {
debug!("No static `PKG-INFO` available for: {source} ({err:?})");
}
Err(err) => return Err(err),
}
// Attempt to read static metadata from the `egg-info` directory.
match read_egg_info(source_root, subdirectory).await {
Ok(metadata) => {
debug!("Found static `egg-info` for: {source}");
// Validate the metadata.
validate(source, &metadata)?;
return Ok(Some(metadata));
}
Err(
err @ (Error::MissingEggInfo
| Error::MissingRequiresTxt
| Error::MissingPkgInfo
| Error::RequiresTxt(
pypi_types::MetadataError::Pep508Error(_)
| pypi_types::MetadataError::RequiresTxtContents(_),
)
| Error::PkgInfo(
pypi_types::MetadataError::Pep508Error(_)
| pypi_types::MetadataError::DynamicField(_)
| pypi_types::MetadataError::FieldNotFound(_)
| pypi_types::MetadataError::UnsupportedMetadataVersion(_),
)),
) => {
debug!("No static `egg-info` available for: {source} ({err:?})");
}
Err(err) => return Err(err),
}
Ok(None)
}
@ -1667,6 +1695,105 @@ impl LocalRevisionPointer {
}
}
/// Read the [`Metadata23`] by combining a source distribution's `PKG-INFO` file with a
/// `requires.txt`.
///
/// `requires.txt` is a legacy concept from setuptools. For example, here's
/// `Flask.egg-info/requires.txt` from Flask's 1.0 release:
///
/// ```txt
/// Werkzeug>=0.14
/// Jinja2>=2.10
/// itsdangerous>=0.24
/// click>=5.1
///
/// [dev]
/// pytest>=3
/// coverage
/// tox
/// sphinx
/// pallets-sphinx-themes
/// sphinxcontrib-log-cabinet
///
/// [docs]
/// sphinx
/// pallets-sphinx-themes
/// sphinxcontrib-log-cabinet
///
/// [dotenv]
/// python-dotenv
/// ```
///
/// See: <https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#dependency-metadata>
async fn read_egg_info(
source_tree: &Path,
subdirectory: Option<&Path>,
) -> Result<Metadata23, Error> {
fn find_egg_info(source_tree: &Path) -> std::io::Result<Option<PathBuf>> {
for entry in fs_err::read_dir(source_tree)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
let path = entry.path();
if path
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("egg-info"))
{
return Ok(Some(path));
}
}
}
Ok(None)
}
let directory = match subdirectory {
Some(subdirectory) => Cow::Owned(source_tree.join(subdirectory)),
None => Cow::Borrowed(source_tree),
};
// Locate the `egg-info` directory.
let egg_info = match find_egg_info(directory.as_ref()) {
Ok(Some(path)) => path,
Ok(None) => return Err(Error::MissingEggInfo),
Err(err) => return Err(Error::CacheRead(err)),
};
// Read the `requires.txt`.
let requires_txt = egg_info.join("requires.txt");
let content = match fs::read(requires_txt).await {
Ok(content) => content,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::MissingRequiresTxt);
}
Err(err) => return Err(Error::CacheRead(err)),
};
// Parse the `requires.txt.
let requires_txt = RequiresTxt::parse(&content).map_err(Error::RequiresTxt)?;
// Read the `PKG-INFO` file.
let pkg_info = egg_info.join("PKG-INFO");
let content = match fs::read(pkg_info).await {
Ok(content) => content,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::MissingPkgInfo);
}
Err(err) => return Err(Error::CacheRead(err)),
};
// Parse the metadata.
let metadata = Metadata12::parse_metadata(&content).map_err(Error::PkgInfo)?;
// Combine the sources.
Ok(Metadata23 {
name: metadata.name,
version: metadata.version,
requires_python: metadata.requires_python,
requires_dist: requires_txt.requires_dist,
provides_extras: requires_txt.provides_extras,
})
}
/// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2
/// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and
/// `Provides-Extra`) are marked as dynamic.

38
crates/uv/tests/pip_compile.rs
View file

@ -11995,3 +11995,41 @@ fn universal_constrained_environment() -> Result<()> {
Ok(())
}
/// Resolve a version of Flask that ships a `requires.txt` file in an `egg-info` directory, but
/// otherwise doesn't include static metadata.
#[test]
fn compile_requires_txt() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("flask @ https://files.pythonhosted.org/packages/36/70/2234ee8842148cef44261c2cebca3a6384894bce6112b73b18693cdcc62f/Flask-1.0.4.tar.gz")?;
uv_snapshot!(context
.pip_compile()
.arg("requirements.in"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] requirements.in
click==8.1.7
# via flask
flask @ https://files.pythonhosted.org/packages/36/70/2234ee8842148cef44261c2cebca3a6384894bce6112b73b18693cdcc62f/Flask-1.0.4.tar.gz
# via -r requirements.in
itsdangerous==2.1.2
# via flask
jinja2==3.1.3
# via flask
markupsafe==2.1.5
# via
# jinja2
# werkzeug
werkzeug==3.0.1
# via flask
----- stderr -----
Resolved 6 packages in [TIME]
"###);
Ok(())
}