Move unnamed requirements to their own pep508_rs module and requirements-txt (#3186)

Another refactoring in preparation of using a richer requirements type.
No functional changes, only moves code around
This commit is contained in:
konsti 2024-04-22 16:02:39 +02:00 committed by GitHub
parent f29c991e21
commit 82c4772e89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 262 additions and 244 deletions

1
Cargo.lock generated
View file

@ -4979,6 +4979,7 @@ dependencies = [
"pep440_rs",
"pep508_rs",
"pypi-types",
"requirements-txt",
"rustc-hash",
"serde",
"serde_json",

View file

@ -45,10 +45,14 @@ use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use uv_fs::normalize_url_path;
// Parity with the crates.io version of pep508_rs
use crate::verbatim_url::VerbatimUrlError;
#[cfg(feature = "non-pep508-extensions")]
pub use unnamed::UnnamedRequirement;
pub use uv_normalize::{ExtraName, InvalidNameError, PackageName};
pub use verbatim_url::{expand_env_vars, split_scheme, strip_host, Scheme, VerbatimUrl};
mod marker;
#[cfg(feature = "non-pep508-extensions")]
mod unnamed;
mod verbatim_url;
/// Error with a span attached. Not that those aren't `String` but `Vec<char>` indices.
@ -122,88 +126,6 @@ create_exception!(
"A PEP 508 parser error with span information"
);
/// A requirement specifier in a `requirements.txt` file.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum RequirementsTxtRequirement {
/// A PEP 508-compliant dependency specifier.
Pep508(Requirement),
/// A PEP 508-like, direct URL dependency specifier.
Unnamed(UnnamedRequirement),
}
impl Display for RequirementsTxtRequirement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Pep508(requirement) => write!(f, "{requirement}"),
Self::Unnamed(requirement) => write!(f, "{requirement}"),
}
}
}
/// A PEP 508-like, direct URL dependency specifier without a package name.
///
/// In a `requirements.txt` file, the name of the package is optional for direct URL
/// dependencies. This isn't compliant with PEP 508, but is common in `requirements.txt`, which
/// is implementation-defined.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "pyo3", pyclass(module = "pep508"))]
pub struct UnnamedRequirement {
/// The direct URL that defines the version specifier.
pub url: VerbatimUrl,
/// The list of extras such as `security`, `tests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
pub extras: Vec<ExtraName>,
/// The markers such as `python_version > "3.8"` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
/// Those are a nested and/or tree.
pub marker: Option<MarkerTree>,
}
impl Display for UnnamedRequirement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.url)?;
if !self.extras.is_empty() {
write!(
f,
"[{}]",
self.extras
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(",")
)?;
}
if let Some(marker) = &self.marker {
write!(f, " ; {}", marker)?;
}
Ok(())
}
}
/// <https://github.com/serde-rs/serde/issues/908#issuecomment-298027413>
#[cfg(feature = "serde")]
impl<'de> Deserialize<'de> for UnnamedRequirement {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
FromStr::from_str(&s).map_err(de::Error::custom)
}
}
/// <https://github.com/serde-rs/serde/issues/1316#issue-332908452>
#[cfg(feature = "serde")]
impl Serialize for UnnamedRequirement {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.collect_str(self)
}
}
/// A PEP 508 dependency specifier.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "pyo3", pyclass(module = "pep508"))]
@ -499,69 +421,6 @@ impl Requirement {
}
}
impl UnnamedRequirement {
/// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool {
if let Some(marker) = &self.marker {
marker.evaluate(env, extras)
} else {
true
}
}
}
impl RequirementsTxtRequirement {
/// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool {
match self {
Self::Pep508(requirement) => requirement.evaluate_markers(env, extras),
Self::Unnamed(requirement) => requirement.evaluate_markers(env, extras),
}
}
/// Returns the extras for the requirement.
pub fn extras(&self) -> &[ExtraName] {
match self {
Self::Pep508(requirement) => requirement.extras.as_slice(),
Self::Unnamed(requirement) => requirement.extras.as_slice(),
}
}
/// Returns the markers for the requirement.
pub fn markers(&self) -> Option<&MarkerTree> {
match self {
Self::Pep508(requirement) => requirement.marker.as_ref(),
Self::Unnamed(requirement) => requirement.marker.as_ref(),
}
}
/// Return the version specifier or URL for the requirement.
pub fn version_or_url(&self) -> Option<VersionOrUrlRef> {
match self {
Self::Pep508(requirement) => match requirement.version_or_url.as_ref() {
Some(VersionOrUrl::VersionSpecifier(specifiers)) => {
Some(VersionOrUrlRef::VersionSpecifier(specifiers))
}
Some(VersionOrUrl::Url(url)) => Some(VersionOrUrlRef::Url(url)),
None => None,
},
Self::Unnamed(requirement) => Some(VersionOrUrlRef::Url(&requirement.url)),
}
}
}
impl From<Requirement> for RequirementsTxtRequirement {
fn from(requirement: Requirement) -> Self {
Self::Pep508(requirement)
}
}
impl From<UnnamedRequirement> for RequirementsTxtRequirement {
fn from(requirement: UnnamedRequirement) -> Self {
Self::Unnamed(requirement)
}
}
impl FromStr for Requirement {
type Err = Pep508Error;
@ -578,59 +437,6 @@ impl Requirement {
}
}
impl FromStr for UnnamedRequirement {
type Err = Pep508Error;
/// Parse a PEP 508-like direct URL requirement without a package name.
fn from_str(input: &str) -> Result<Self, Self::Err> {
parse_unnamed_requirement(&mut Cursor::new(input), None)
}
}
impl UnnamedRequirement {
/// Parse a PEP 508-like direct URL requirement without a package name.
pub fn parse(input: &str, working_dir: impl AsRef<Path>) -> Result<Self, Pep508Error> {
parse_unnamed_requirement(&mut Cursor::new(input), Some(working_dir.as_ref()))
}
}
impl FromStr for RequirementsTxtRequirement {
type Err = Pep508Error;
/// Parse a requirement as seen in a `requirements.txt` file.
fn from_str(input: &str) -> Result<Self, Self::Err> {
match Requirement::from_str(input) {
Ok(requirement) => Ok(Self::Pep508(requirement)),
Err(err) => match err.message {
Pep508ErrorSource::UnsupportedRequirement(_) => {
Ok(Self::Unnamed(UnnamedRequirement::from_str(input)?))
}
_ => Err(err),
},
}
}
}
impl RequirementsTxtRequirement {
/// Parse a requirement as seen in a `requirements.txt` file.
pub fn parse(input: &str, working_dir: impl AsRef<Path>) -> Result<Self, Pep508Error> {
// Attempt to parse as a PEP 508-compliant requirement.
match Requirement::parse(input, &working_dir) {
Ok(requirement) => Ok(Self::Pep508(requirement)),
Err(err) => match err.message {
Pep508ErrorSource::UnsupportedRequirement(_) => {
// If that fails, attempt to parse as a direct URL requirement.
Ok(Self::Unnamed(UnnamedRequirement::parse(
input,
&working_dir,
)?))
}
_ => Err(err),
},
}
}
}
/// A list of [`ExtraName`] that can be attached to a [`Requirement`].
#[derive(Debug, Clone, Eq, Hash, PartialEq)]
pub struct Extras(Vec<ExtraName>);
@ -818,8 +624,8 @@ fn parse_name(cursor: &mut Cursor) -> Result<PackageName, Pep508Error> {
} else {
Err(Pep508Error {
message: Pep508ErrorSource::String(format!(
"Expected package name starting with an alphanumeric character, found '{char}'"
)),
"Expected package name starting with an alphanumeric character, found '{char}'"
)),
start: index,
len: char.len_utf8(),
input: cursor.to_string(),
@ -928,7 +734,7 @@ fn parse_extras(cursor: &mut Cursor) -> Result<Vec<ExtraName>, Pep508Error> {
(Some((pos, other)), false) => {
return Err(Pep508Error {
message: Pep508ErrorSource::String(
format!("Expected either ',' (separating extras) or ']' (ending the extras section), found '{other}'",)
format!("Expected either ',' (separating extras) or ']' (ending the extras section), found '{other}'")
),
start: pos,
len: 1,
@ -1592,7 +1398,8 @@ mod tests {
parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
MarkerValueString, MarkerValueVersion,
};
use crate::{Cursor, Pep508Error, Requirement, UnnamedRequirement, VerbatimUrl, VersionOrUrl};
use crate::unnamed::UnnamedRequirement;
use crate::{Cursor, Pep508Error, Requirement, VerbatimUrl, VersionOrUrl};
fn parse_pepe508_err(input: &str) -> String {
Requirement::from_str(input).unwrap_err().to_string()

View file

@ -0,0 +1,101 @@
use std::fmt::{Display, Formatter};
use std::path::Path;
use std::str::FromStr;
#[cfg(feature = "pyo3")]
use pyo3::pyclass;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use uv_normalize::ExtraName;
use crate::{Cursor, MarkerEnvironment, MarkerTree, Pep508Error, VerbatimUrl};
/// A PEP 508-like, direct URL dependency specifier without a package name.
///
/// In a `requirements.txt` file, the name of the package is optional for direct URL
/// dependencies. This isn't compliant with PEP 508, but is common in `requirements.txt`, which
/// is implementation-defined.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "pyo3", pyclass(module = "pep508"))]
pub struct UnnamedRequirement {
/// The direct URL that defines the version specifier.
pub url: VerbatimUrl,
/// The list of extras such as `security`, `tests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
pub extras: Vec<ExtraName>,
/// The markers such as `python_version > "3.8"` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
/// Those are a nested and/or tree.
pub marker: Option<MarkerTree>,
}
impl UnnamedRequirement {
/// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool {
if let Some(marker) = &self.marker {
marker.evaluate(env, extras)
} else {
true
}
}
}
impl Display for UnnamedRequirement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.url)?;
if !self.extras.is_empty() {
write!(
f,
"[{}]",
self.extras
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(",")
)?;
}
if let Some(marker) = &self.marker {
write!(f, " ; {}", marker)?;
}
Ok(())
}
}
/// <https://github.com/serde-rs/serde/issues/908#issuecomment-298027413>
#[cfg(feature = "serde")]
impl<'de> Deserialize<'de> for UnnamedRequirement {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
FromStr::from_str(&s).map_err(de::Error::custom)
}
}
/// <https://github.com/serde-rs/serde/issues/1316#issue-332908452>
#[cfg(feature = "serde")]
impl Serialize for UnnamedRequirement {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.collect_str(self)
}
}
impl FromStr for UnnamedRequirement {
type Err = Pep508Error;
/// Parse a PEP 508-like direct URL requirement without a package name.
fn from_str(input: &str) -> Result<Self, Self::Err> {
crate::parse_unnamed_requirement(&mut Cursor::new(input), None)
}
}
impl UnnamedRequirement {
/// Parse a PEP 508-like direct URL requirement without a package name.
pub fn parse(input: &str, working_dir: impl AsRef<Path>) -> Result<Self, Pep508Error> {
crate::parse_unnamed_requirement(&mut Cursor::new(input), Some(working_dir.as_ref()))
}
}

View file

@ -46,8 +46,9 @@ use unscanny::{Pattern, Scanner};
use url::Url;
use pep508_rs::{
expand_env_vars, split_scheme, strip_host, Extras, Pep508Error, Pep508ErrorSource, Requirement,
RequirementsTxtRequirement, Scheme, VerbatimUrl,
expand_env_vars, split_scheme, strip_host, Extras, MarkerEnvironment, MarkerTree, Pep508Error,
Pep508ErrorSource, Requirement, Scheme, UnnamedRequirement, VerbatimUrl, VersionOrUrl,
VersionOrUrlRef,
};
#[cfg(feature = "http")]
use uv_client::BaseClient;
@ -2183,3 +2184,110 @@ mod test {
assert_eq!(line_column, expected, "Issues with input: {input}");
}
}
/// A requirement specifier in a `requirements.txt` file.
#[derive(Hash, Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub enum RequirementsTxtRequirement {
/// A PEP 508-compliant dependency specifier.
Pep508(Requirement),
/// A PEP 508-like, direct URL dependency specifier.
Unnamed(UnnamedRequirement),
}
impl Display for RequirementsTxtRequirement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Pep508(requirement) => write!(f, "{requirement}"),
Self::Unnamed(requirement) => write!(f, "{requirement}"),
}
}
}
impl RequirementsTxtRequirement {
/// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool {
match self {
Self::Pep508(requirement) => requirement.evaluate_markers(env, extras),
Self::Unnamed(requirement) => requirement.evaluate_markers(env, extras),
}
}
/// Returns the extras for the requirement.
pub fn extras(&self) -> &[ExtraName] {
match self {
Self::Pep508(requirement) => requirement.extras.as_slice(),
Self::Unnamed(requirement) => requirement.extras.as_slice(),
}
}
/// Returns the markers for the requirement.
pub fn markers(&self) -> Option<&MarkerTree> {
match self {
Self::Pep508(requirement) => requirement.marker.as_ref(),
Self::Unnamed(requirement) => requirement.marker.as_ref(),
}
}
/// Return the version specifier or URL for the requirement.
pub fn version_or_url(&self) -> Option<VersionOrUrlRef> {
match self {
Self::Pep508(requirement) => match requirement.version_or_url.as_ref() {
Some(VersionOrUrl::VersionSpecifier(specifiers)) => {
Some(VersionOrUrlRef::VersionSpecifier(specifiers))
}
Some(VersionOrUrl::Url(url)) => Some(VersionOrUrlRef::Url(url)),
None => None,
},
Self::Unnamed(requirement) => Some(VersionOrUrlRef::Url(&requirement.url)),
}
}
}
impl From<Requirement> for RequirementsTxtRequirement {
fn from(requirement: Requirement) -> Self {
Self::Pep508(requirement)
}
}
impl From<UnnamedRequirement> for RequirementsTxtRequirement {
fn from(requirement: UnnamedRequirement) -> Self {
Self::Unnamed(requirement)
}
}
impl FromStr for RequirementsTxtRequirement {
type Err = Pep508Error;
/// Parse a requirement as seen in a `requirements.txt` file.
fn from_str(input: &str) -> Result<Self, Self::Err> {
match Requirement::from_str(input) {
Ok(requirement) => Ok(Self::Pep508(requirement)),
Err(err) => match err.message {
Pep508ErrorSource::UnsupportedRequirement(_) => {
Ok(Self::Unnamed(UnnamedRequirement::from_str(input)?))
}
_ => Err(err),
},
}
}
}
impl RequirementsTxtRequirement {
/// Parse a requirement as seen in a `requirements.txt` file.
pub fn parse(input: &str, working_dir: impl AsRef<Path>) -> Result<Self, Pep508Error> {
// Attempt to parse as a PEP 508-compliant requirement.
match Requirement::parse(input, &working_dir) {
Ok(requirement) => Ok(Self::Pep508(requirement)),
Err(err) => match err.message {
Pep508ErrorSource::UnsupportedRequirement(_) => {
// If that fails, attempt to parse as a direct URL requirement.
Ok(Self::Unnamed(UnnamedRequirement::parse(
input,
&working_dir,
)?))
}
_ => Err(err),
},
}
}
}

View file

@ -9,8 +9,8 @@ use url::Url;
use distribution_types::{InstalledDist, InstalledMetadata, InstalledVersion, Name};
use pep440_rs::{Version, VersionSpecifiers};
use pep508_rs::{Requirement, RequirementsTxtRequirement, VerbatimUrl};
use requirements_txt::{EditableRequirement, RequirementEntry};
use pep508_rs::{Requirement, VerbatimUrl};
use requirements_txt::{EditableRequirement, RequirementEntry, RequirementsTxtRequirement};
use uv_cache::{ArchiveTarget, ArchiveTimestamp};
use uv_interpreter::PythonEnvironment;
use uv_normalize::PackageName;
@ -570,14 +570,14 @@ impl Diagnostic {
} => format!(
"The package `{package}` requires `{requirement}`, but `{version}` is installed."
),
Self::DuplicatePackage { package, paths} => {
Self::DuplicatePackage { package, paths } => {
let mut paths = paths.clone();
paths.sort();
format!(
"The package `{package}` has multiple installed distributions:{}",
paths.iter().fold(String::new(), |acc, path| acc + &format!("\n - {}", path.display()))
)
},
}
}
}

View file

@ -2,12 +2,14 @@ use std::path::PathBuf;
use anyhow::{Context, Result};
use rustc_hash::FxHashSet;
use tracing::{instrument, Level};
use tracing::instrument;
use cache_key::CanonicalUrl;
use distribution_types::{FlatIndexLocation, IndexUrl};
use pep508_rs::{Requirement, RequirementsTxtRequirement};
use requirements_txt::{EditableRequirement, FindLink, RequirementEntry, RequirementsTxt};
use pep508_rs::Requirement;
use requirements_txt::{
EditableRequirement, FindLink, RequirementEntry, RequirementsTxt, RequirementsTxtRequirement,
};
use uv_client::BaseClientBuilder;
use uv_configuration::{NoBinary, NoBuild};
use uv_fs::Simplified;
@ -48,7 +50,7 @@ pub struct RequirementsSpecification {
impl RequirementsSpecification {
/// Read the requirements and constraints from a source.
#[instrument(skip_all, level = Level::DEBUG, fields(source = % source))]
#[instrument(skip_all, level = tracing::Level::DEBUG, fields(source = % source))]
pub async fn from_source(
source: &RequirementsSource,
extras: &ExtrasSpecification,

View file

@ -13,11 +13,9 @@ use distribution_types::{
BuildableSource, DirectSourceUrl, GitSourceUrl, PathSourceUrl, RemoteSource, SourceUrl,
VersionId,
};
use pep508_rs::{
Requirement, RequirementsTxtRequirement, Scheme, UnnamedRequirement, VersionOrUrl,
};
use pep508_rs::{Requirement, Scheme, UnnamedRequirement, VersionOrUrl};
use pypi_types::Metadata10;
use requirements_txt::RequirementEntry;
use requirements_txt::{RequirementEntry, RequirementsTxtRequirement};
use uv_client::RegistryClient;
use uv_distribution::{DistributionDatabase, Reporter};
use uv_normalize::PackageName;

View file

@ -1,14 +1,13 @@
use std::str::FromStr;
use rustc_hash::FxHashMap;
use tracing::trace;
use pep440_rs::{Operator, Version};
use pep508_rs::{
MarkerEnvironment, Requirement, RequirementsTxtRequirement, UnnamedRequirement, VersionOrUrl,
};
use pep508_rs::UnnamedRequirement;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use pypi_types::{HashDigest, HashError};
use requirements_txt::RequirementEntry;
use tracing::trace;
use requirements_txt::{RequirementEntry, RequirementsTxtRequirement};
use uv_normalize::PackageName;
#[derive(thiserror::Error, Debug)]
@ -33,7 +32,7 @@ impl Preference {
requirement: match entry.requirement {
RequirementsTxtRequirement::Pep508(requirement) => requirement,
RequirementsTxtRequirement::Unnamed(requirement) => {
return Err(PreferenceError::Bare(requirement))
return Err(PreferenceError::Bare(requirement));
}
},
hashes: entry
@ -99,27 +98,27 @@ impl Preferences {
}
match requirement.version_or_url.as_ref() {
Some(VersionOrUrl::VersionSpecifier(version_specifiers)) =>
{
let [version_specifier] = version_specifiers.as_ref() else {
trace!(
{
let [version_specifier] = version_specifiers.as_ref() else {
trace!(
"Excluding {requirement} from preferences due to multiple version specifiers."
);
return None;
};
if *version_specifier.operator() != Operator::Equal {
trace!(
return None;
};
if *version_specifier.operator() != Operator::Equal {
trace!(
"Excluding {requirement} from preferences due to inexact version specifier."
);
return None;
return None;
}
Some((
requirement.name,
Pin {
version: version_specifier.version().clone(),
hashes,
},
))
}
Some((
requirement.name,
Pin {
version: version_specifier.version().clone(),
hashes,
},
))
}
Some(VersionOrUrl::Url(_)) => {
trace!(
"Excluding {requirement} from preferences due to URL dependency."
@ -127,10 +126,9 @@ impl Preferences {
None
}
_ => {
None
None
}
}
}
})
.collect(),
)

View file

@ -18,6 +18,7 @@ once-map = { workspace = true }
pep440_rs = { workspace = true }
pep508_rs = { workspace = true }
pypi-types = { workspace = true }
requirements-txt = { workspace = true }
uv-cache = { workspace = true }
uv-interpreter = { workspace = true }
uv-normalize = { workspace = true }

View file

@ -4,8 +4,9 @@ use rustc_hash::FxHashMap;
use url::Url;
use distribution_types::{DistributionMetadata, HashPolicy, PackageId};
use pep508_rs::{MarkerEnvironment, RequirementsTxtRequirement, VersionOrUrl};
use pep508_rs::{MarkerEnvironment, VersionOrUrl};
use pypi_types::{HashDigest, HashError};
use requirements_txt::RequirementsTxtRequirement;
use uv_normalize::PackageName;
#[derive(Debug, Clone)]
@ -122,7 +123,7 @@ impl HashStrategy {
None => {
return Err(HashStrategyError::UnpinnedRequirement(
requirement.to_string(),
))
));
}
}
}

View file

@ -6,7 +6,8 @@ use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::{InstalledMetadata, Name};
use pep508_rs::{Requirement, RequirementsTxtRequirement, UnnamedRequirement};
use pep508_rs::{Requirement, UnnamedRequirement};
use requirements_txt::RequirementsTxtRequirement;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity};
use uv_configuration::KeyringProviderType;