mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Wheel filename distribution package name (#278)

Browse source 
The normalized name abstractions were not consistently, this PR uses
them where they were previously missing:
* `WheelFilename::distribution`
* `Requirement::name`
* `Requirement::extras`
* `Metadata21::name`
* `Metadata21::provides_dist`

With `puffin-package` depending on `pep508_rs` this would be cyclical
crate dependency, so `puffin-normalize` gets split out from
`puffin-package`.

`DistInfoName` has the same task and semantics as `PackageName`, so it's
merged into the latter.

`PackageName` and `ExtraName` documentation is moved onto the type and
their constructors are called `new` instead of `normalize`. We now use
these constructors rarely enough the implicit allocation by
`to_string()` shouldn't matter anymore, while more actual cloning
becomes visible.
This commit is contained in:
konsti 2023-11-02 12:15:27 +01:00 • committed by GitHub
parent 8a8b532330
commit 4adaa9a700
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
61 changed files with 483 additions and 416 deletions
Download patch file Download diff file Expand all files Collapse all files

27
Cargo.lock generated
View file

@ -704,7 +704,7 @@ version = "0.0.1"
dependencies = [
"pep440_rs 0.3.12",
"platform-tags",
"puffin-package",
"puffin-normalize",
"thiserror",
"url",
]
@ -1737,6 +1737,7 @@ dependencies = [
"log",
"once_cell",
"pep440_rs 0.3.12",
"puffin-normalize",
"pyo3",
"pyo3-log",
"regex",
@ -2010,6 +2011,7 @@ dependencies = [
"puffin-distribution",
"puffin-installer",
"puffin-interpreter",
"puffin-normalize",
"puffin-package",
"puffin-resolver",
"puffin-workspace",
@ -2031,6 +2033,7 @@ version = "0.0.1"
dependencies = [
"futures",
"http-cache-reqwest",
"puffin-normalize",
"puffin-package",
"reqwest",
"reqwest-middleware",
@ -2102,6 +2105,7 @@ dependencies = [
"anyhow",
"pep440_rs 0.3.12",
"puffin-cache",
"puffin-normalize",
"puffin-package",
"url",
]
@ -2121,6 +2125,7 @@ dependencies = [
"puffin-client",
"puffin-distribution",
"puffin-interpreter",
"puffin-normalize",
"puffin-package",
"rayon",
"tempfile",
@ -2148,6 +2153,21 @@ dependencies = [
"tracing",
]
[[package]]
name = "puffin-normalize"
version = "0.0.1"
dependencies = [
"anyhow",
"indoc 2.0.4",
"insta",
"once_cell",
"regex",
"serde",
"serde_json",
"tempfile",
"test-case",
]
[[package]]
name = "puffin-package"
version = "0.0.1"
@ -2157,10 +2177,10 @@ dependencies = [
"indoc 2.0.4",
"insta",
"mailparse",
"memchr",
"once_cell",
"pep440_rs 0.3.12",
"pep508_rs",
"puffin-normalize",
"regex",
"rfc2047-decoder",
"serde",
@ -2199,6 +2219,7 @@ dependencies = [
"puffin-client",
"puffin-distribution",
"puffin-interpreter",
"puffin-normalize",
"puffin-package",
"puffin-traits",
"sha2",
@ -2228,7 +2249,7 @@ dependencies = [
"fs-err",
"pep440_rs 0.3.12",
"pep508_rs",
"puffin-package",
"puffin-normalize",
"pyproject-toml",
"serde",
"thiserror",

1
Cargo.toml
View file

@ -36,7 +36,6 @@ indicatif = { version = "0.17.7" }
indoc = { version = "2.0.4" }
itertools = { version = "0.11.0" }
mailparse = { version = "0.14.0" }
memchr = { version = "2.6.4" }
miette = { version = "5.10.0" }
once_cell = { version = "1.18.0" }
petgraph = { version = "0.6.4" }

2
crates/distribution-filename/Cargo.toml
View file

@ -12,7 +12,7 @@ license = { workspace = true }
[dependencies]
pep440_rs = { path = "../pep440-rs" }
platform-tags = { path = "../platform-tags" }
puffin-package = { path = "../puffin-package" }
puffin-normalize = { path = "../puffin-normalize" }
thiserror = { workspace = true }
url = { workspace = true }

17
crates/distribution-filename/src/source_distribution.rs
View file

@ -1,9 +1,11 @@
use pep440_rs::Version;
use puffin_package::package_name::PackageName;
use std::fmt::{Display, Formatter};
use std::str::FromStr;
use thiserror::Error;
use pep440_rs::Version;
use puffin_normalize::PackageName;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum SourceDistributionExtension {
Zip,
@ -70,7 +72,7 @@ impl SourceDistributionFilename {
};
if stem.len() <= package_name.as_ref().len() + "-".len()
|| &PackageName::normalize(&stem[..package_name.as_ref().len()]) != package_name
|| &PackageName::new(&stem[..package_name.as_ref().len()]) != package_name
{
return Err(SourceDistributionFilenameError::InvalidFilename {
filename: filename.to_string(),
@ -111,8 +113,9 @@ pub enum SourceDistributionFilenameError {
#[cfg(test)]
mod tests {
use puffin_normalize::PackageName;
use crate::SourceDistributionFilename;
use puffin_package::package_name::PackageName;
/// Only test already normalized names since the parsing is lossy
#[test]
@ -123,7 +126,7 @@ mod tests {
"foo-lib-1.2.3.tar.gz",
] {
assert_eq!(
SourceDistributionFilename::parse(normalized, &PackageName::normalize("foo_lib"))
SourceDistributionFilename::parse(normalized, &PackageName::new("foo_lib"))
.unwrap()
.to_string(),
normalized
@ -134,9 +137,7 @@ mod tests {
#[test]
fn errors() {
for invalid in ["b-1.2.3.zip", "a-1.2.3-gamma.3.zip", "a-1.2.3.tar.zstd"] {
assert!(
SourceDistributionFilename::parse(invalid, &PackageName::normalize("a")).is_err()
);
assert!(SourceDistributionFilename::parse(invalid, &PackageName::new("a")).is_err());
}
}
}

7
crates/distribution-filename/src/wheel.rs
View file

@ -1,15 +1,16 @@
use std::fmt::{Display, Formatter};
use std::str::FromStr;
use pep440_rs::Version;
use thiserror::Error;
use url::Url;
use pep440_rs::Version;
use platform_tags::Tags;
use puffin_normalize::PackageName;
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct WheelFilename {
pub distribution: String,
pub distribution: PackageName,
pub version: Version,
pub python_tag: Vec<String>,
pub abi_tag: Vec<String>,
@ -89,7 +90,7 @@ impl FromStr for WheelFilename {
let version = Version::from_str(version)
.map_err(|err| WheelFilenameError::InvalidVersion(filename.to_string(), err))?;
Ok(WheelFilename {
distribution: distribution.to_string(),
distribution: PackageName::new(distribution),
version,
python_tag: python_tag.split('.').map(String::from).collect(),
abi_tag: abi_tag.split('.').map(String::from).collect(),

6
crates/install-wheel-rs/src/wheel.rs
View file

@ -899,7 +899,7 @@ pub fn install_wheel(
sys_executable: impl AsRef<Path>,
) -> Result<String, Error> {
let name = &filename.distribution;
let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_str());
let _my_span = span!(Level::DEBUG, "install_wheel", name = name.as_ref());
let base_location = location.venv_root();
@ -918,12 +918,12 @@ pub fn install_wheel(
.join("site-packages")
};
debug!(name = name.as_str(), "Opening zip");
debug!(name = name.as_ref(), "Opening zip");
// No BufReader: https://github.com/zip-rs/zip/issues/381
let mut archive =
ZipArchive::new(reader).map_err(|err| Error::from_zip_error("(index)".to_string(), err))?;
debug!(name = name.as_str(), "Getting wheel metadata");
debug!(name = name.as_ref(), "Getting wheel metadata");
let dist_info_prefix = find_dist_info(filename, &mut archive)?;
let (name, _version) = read_metadata(&dist_info_prefix, &mut archive)?;
// TODO: Check that name and version match

1
crates/pep508-rs/Cargo.toml
View file

@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
pep440_rs = { path = "../pep440-rs" }
puffin-normalize = { path = "../puffin-normalize" }
once_cell = { workspace = true }
regex = { workspace = true }

100
crates/pep508-rs/src/lib.rs
View file

@ -6,31 +6,16 @@
//! ```
//! use std::str::FromStr;
//! use pep508_rs::Requirement;
//! use puffin_normalize::ExtraName;
//!
//! let marker = r#"requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8""#;
//! let dependency_specification = Requirement::from_str(marker).unwrap();
//! assert_eq!(dependency_specification.name, "requests");
//! assert_eq!(dependency_specification.extras, Some(vec!["security".to_string(), "tests".to_string()]));
//! assert_eq!(dependency_specification.name.as_ref(), "requests");
//! assert_eq!(dependency_specification.extras, Some(vec![ExtraName::new("security"), ExtraName::new("tests")]));
//! ```
#![deny(missing_docs)]
mod marker;
pub use marker::{
MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
MarkerValueString, MarkerValueVersion, MarkerWarningKind, StringVersion,
};
#[cfg(feature = "pyo3")]
use pep440_rs::PyVersion;
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
#[cfg(feature = "pyo3")]
use pyo3::{
basic::CompareOp, create_exception, exceptions::PyNotImplementedError, pyclass, pymethods,
pymodule, types::PyModule, IntoPy, PyObject, PyResult, Python,
};
#[cfg(feature = "serde")]
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
#[cfg(feature = "pyo3")]
use std::collections::hash_map::DefaultHasher;
use std::collections::HashSet;
@ -38,10 +23,29 @@ use std::fmt::{Display, Formatter};
#[cfg(feature = "pyo3")]
use std::hash::{Hash, Hasher};
use std::str::{Chars, FromStr};
#[cfg(feature = "pyo3")]
use pep440_rs::PyVersion;
#[cfg(feature = "pyo3")]
use pyo3::{
create_exception, exceptions::PyNotImplementedError, pyclass, pyclass::CompareOp, pymethods,
pymodule, types::PyModule, IntoPy, PyObject, PyResult, Python,
};
#[cfg(feature = "serde")]
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use thiserror::Error;
use unicode_width::UnicodeWidthStr;
use url::Url;
pub use marker::{
MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
MarkerValueString, MarkerValueVersion, MarkerWarningKind, StringVersion,
};
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use puffin_normalize::{ExtraName, PackageName};
mod marker;
/// Error with a span attached. Not that those aren't `String` but `Vec<char>` indices.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Pep508Error {
@ -128,10 +132,10 @@ create_exception!(
pub struct Requirement {
/// The distribution name such as `numpy` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
pub name: String,
pub name: PackageName,
/// The list of extras such as `security`, `tests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
pub extras: Option<Vec<String>>,
pub extras: Option<Vec<ExtraName>>,
/// The version specifier such as `>= 2.8.1`, `== 2.8.*` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
/// or a url
@ -146,7 +150,15 @@ impl Display for Requirement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.name)?;
if let Some(extras) = &self.extras {
write!(f, "[{}]", extras.join(","))?;
write!(
f,
"[{}]",
extras
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(",")
)?;
}
if let Some(version_or_url) = &self.version_or_url {
match version_or_url {
@ -198,14 +210,16 @@ impl Requirement {
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
#[getter]
pub fn name(&self) -> String {
self.name.clone()
self.name.to_string()
}
/// The list of extras such as `security`, `tests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
#[getter]
pub fn extras(&self) -> Option<Vec<String>> {
self.extras.clone()
self.extras
.as_ref()
.map(|extras| extras.iter().map(ToString::to_string).collect())
}
/// The marker expression such as `python_version > "3.8"` in
@ -511,7 +525,7 @@ impl<'a> CharIter<'a> {
}
}
fn parse_name(chars: &mut CharIter) -> Result<String, Pep508Error> {
fn parse_name(chars: &mut CharIter) -> Result<PackageName, Pep508Error> {
// https://peps.python.org/pep-0508/#names
// ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$ with re.IGNORECASE
let mut name = String::new();
@ -554,13 +568,13 @@ fn parse_name(chars: &mut CharIter) -> Result<String, Pep508Error> {
});
}
}
Some(_) | None => return Ok(name),
Some(_) | None => return Ok(PackageName::new(name)),
}
}
}
/// parses extras in the `[extra1,extra2] format`
fn parse_extras(chars: &mut CharIter) -> Result<Option<Vec<String>>, Pep508Error> {
fn parse_extras(chars: &mut CharIter) -> Result<Option<Vec<ExtraName>>, Pep508Error> {
let Some(bracket_pos) = chars.eat('[') else {
return Ok(None);
};
@ -627,10 +641,10 @@ fn parse_extras(chars: &mut CharIter) -> Result<Option<Vec<String>>, Pep508Error
// end or next identifier?
match chars.next() {
Some((_, ',')) => {
extras.push(buffer);
extras.push(ExtraName::new(buffer));
}
Some((_, ']')) => {
extras.push(buffer);
extras.push(ExtraName::new(buffer));
break;
}
Some((pos, other)) => {
@ -870,15 +884,19 @@ pub fn python_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
/// Half of these tests are copied from <https://github.com/pypa/packaging/pull/624>
#[cfg(test)]
mod tests {
use std::str::FromStr;
use indoc::indoc;
use url::Url;
use pep440_rs::{Operator, Version, VersionSpecifier};
use puffin_normalize::{ExtraName, PackageName};
use crate::marker::{
parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
MarkerValueString, MarkerValueVersion,
};
use crate::{CharIter, Requirement, VersionOrUrl};
use indoc::indoc;
use pep440_rs::{Operator, Version, VersionSpecifier};
use std::str::FromStr;
use url::Url;
fn assert_err(input: &str, error: &str) {
assert_eq!(Requirement::from_str(input).unwrap_err().to_string(), error);
@ -926,8 +944,8 @@ mod tests {
let requests = Requirement::from_str(input).unwrap();
assert_eq!(input, requests.to_string());
let expected = Requirement {
name: "requests".to_string(),
extras: Some(vec!["security".to_string(), "tests".to_string()]),
name: PackageName::new("requests"),
extras: Some(vec![ExtraName::new("security"), ExtraName::new("tests")]),
version_or_url: Some(VersionOrUrl::VersionSpecifier(
[
VersionSpecifier::new(
@ -972,25 +990,25 @@ mod tests {
#[test]
fn parenthesized_single() {
let numpy = Requirement::from_str("numpy ( >=1.19 )").unwrap();
assert_eq!(numpy.name, "numpy");
assert_eq!(numpy.name.as_ref(), "numpy");
}
#[test]
fn parenthesized_double() {
let numpy = Requirement::from_str("numpy ( >=1.19, <2.0 )").unwrap();
assert_eq!(numpy.name, "numpy");
assert_eq!(numpy.name.as_ref(), "numpy");
}
#[test]
fn versions_single() {
let numpy = Requirement::from_str("numpy >=1.19 ").unwrap();
assert_eq!(numpy.name, "numpy");
assert_eq!(numpy.name.as_ref(), "numpy");
}
#[test]
fn versions_double() {
let numpy = Requirement::from_str("numpy >=1.19, <2.0 ").unwrap();
assert_eq!(numpy.name, "numpy");
assert_eq!(numpy.name.as_ref(), "numpy");
}
#[test]
@ -1068,7 +1086,7 @@ mod tests {
#[test]
fn error_extras1() {
let numpy = Requirement::from_str("black[d]").unwrap();
assert_eq!(numpy.extras, Some(vec!["d".to_string()]));
assert_eq!(numpy.extras, Some(vec![ExtraName::new("d")]));
}
#[test]
@ -1076,7 +1094,7 @@ mod tests {
let numpy = Requirement::from_str("black[d,jupyter]").unwrap();
assert_eq!(
numpy.extras,
Some(vec!["d".to_string(), "jupyter".to_string()])
Some(vec![ExtraName::new("d"), ExtraName::new("jupyter")])
);
}
@ -1123,7 +1141,7 @@ mod tests {
.unwrap();
let url = "https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686";
let expected = Requirement {
name: "pip".to_string(),
name: PackageName::new("pip"),
extras: None,
marker: None,
version_or_url: Some(VersionOrUrl::Url(Url::parse(url).unwrap())),

1
crates/puffin-cli/Cargo.toml
View file

@ -26,6 +26,7 @@ puffin-dispatch = { path = "../puffin-dispatch" }
puffin-distribution = { path = "../puffin-distribution" }
puffin-installer = { path = "../puffin-installer" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-normalize = { path = "../puffin-normalize" }
puffin-package = { path = "../puffin-package" }
puffin-resolver = { path = "../puffin-resolver", features = ["clap"] }
puffin-workspace = { path = "../puffin-workspace" }

3
crates/puffin-cli/src/commands/pip_uninstall.rs
View file

@ -7,7 +7,6 @@ use tracing::debug;
use platform_host::Platform;
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use crate::commands::{elapsed, ExitStatus};
use crate::printer::Printer;
@ -43,7 +42,7 @@ pub(crate) async fn pip_uninstall(
let packages = {
let mut packages = requirements
.into_iter()
.map(|requirement| PackageName::normalize(requirement.name))
.map(|requirement| requirement.name)
.collect::<Vec<_>>();
packages.sort_unstable();
packages.dedup();

6
crates/puffin-cli/src/commands/reporters.rs
View file

@ -3,8 +3,8 @@ use std::time::Duration;
use indicatif::{ProgressBar, ProgressStyle};
use puffin_distribution::{CachedDistribution, RemoteDistribution, VersionOrUrl};
use puffin_package::dist_info_name::DistInfoName;
use puffin_package::package_name::PackageName;
use puffin_normalize::ExtraName;
use puffin_normalize::PackageName;
use crate::printer::Printer;
@ -171,7 +171,7 @@ impl puffin_resolver::ResolverReporter for ResolverReporter {
fn on_progress(
&self,
name: &PackageName,
extra: Option<&DistInfoName>,
extra: Option<&ExtraName>,
version_or_url: VersionOrUrl,
) {
match (extra, version_or_url) {

5
crates/puffin-cli/src/main.rs
View file

@ -4,10 +4,11 @@ use std::process::ExitCode;
use clap::{Args, Parser, Subcommand};
use colored::Colorize;
use directories::ProjectDirs;
use puffin_package::extra_name::ExtraName;
use url::Url;
use puffin_normalize::ExtraName;
use puffin_resolver::{PreReleaseMode, ResolutionMode};
use requirements::ExtrasSpecification;
use url::Url;
use crate::commands::ExitStatus;
use crate::index_urls::IndexUrls;

4
crates/puffin-cli/src/requirements.rs
View file

@ -8,7 +8,7 @@ use anyhow::{Context, Result};
use fs_err as fs;
use pep508_rs::Requirement;
use puffin_package::extra_name::ExtraName;
use puffin_normalize::ExtraName;
use puffin_package::requirements_txt::RequirementsTxt;
#[derive(Debug)]
@ -107,7 +107,7 @@ impl RequirementsSpecification {
for (name, optional_requirements) in
project.optional_dependencies.unwrap_or_default()
{
let normalized_name = ExtraName::normalize(name);
let normalized_name = ExtraName::new(name);
if extras.contains(&normalized_name) {
used_extras.insert(normalized_name);
requirements.extend(optional_requirements);

1
crates/puffin-client/Cargo.toml
View file

@ -4,6 +4,7 @@ version = "0.0.1"
edition = "2021"
[dependencies]
puffin-normalize = { path = "../puffin-normalize" }
puffin-package = { path = "../puffin-package" }
futures = { workspace = true }

8
crates/puffin-client/src/client.rs
View file

@ -11,7 +11,7 @@ use reqwest_retry::RetryTransientMiddleware;
use tracing::trace;
use url::Url;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use puffin_package::pypi_types::{File, Metadata21, SimpleJson};
use crate::error::Error;
@ -135,7 +135,7 @@ pub struct RegistryClient {
impl RegistryClient {
/// Fetch a package from the `PyPI` simple API.
pub async fn simple(&self, package_name: impl AsRef<str>) -> Result<SimpleJson, Error> {
pub async fn simple(&self, package_name: PackageName) -> Result<SimpleJson, Error> {
if self.no_index {
return Err(Error::NoIndex(package_name.as_ref().to_string()));
}
@ -143,9 +143,7 @@ impl RegistryClient {
for index in std::iter::once(&self.index).chain(self.extra_index.iter()) {
// Format the URL for PyPI.
let mut url = index.clone();
url.path_segments_mut()
.unwrap()
.push(PackageName::normalize(&package_name).as_ref());
url.path_segments_mut().unwrap().push(package_name.as_ref());
url.path_segments_mut().unwrap().push("");
url.set_query(Some("format=application/vnd.pypi.simple.v1+json"));

1
crates/puffin-distribution/Cargo.toml
View file

@ -12,6 +12,7 @@ license = { workspace = true }
[dependencies]
pep440_rs = { path = "../pep440-rs" }
puffin-cache = { path = "../puffin-cache" }
puffin-normalize = { path = "../puffin-normalize" }
puffin-package = { path = "../puffin-package" }
anyhow = { workspace = true }

15
crates/puffin-distribution/src/lib.rs
View file

@ -7,8 +7,7 @@ use url::Url;
use pep440_rs::Version;
use puffin_cache::CanonicalUrl;
use puffin_package::dist_info_name::DistInfoName;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use puffin_package::pypi_types::File;
/// A built distribution (wheel), which either exists remotely or locally.
@ -117,7 +116,9 @@ impl RemoteDistribution {
pub fn id(&self) -> String {
match self {
Self::Registry(name, version, _) => {
format!("{}-{}", DistInfoName::from(name), version)
// https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory
// `version` is normalized by its `ToString` impl
format!("{}-{}", PackageName::from(name), version)
}
Self::Url(_name, url) => puffin_cache::digest(&CanonicalUrl::new(url)),
}
@ -169,7 +170,7 @@ impl CachedDistribution {
return Ok(None);
};
let name = PackageName::normalize(name);
let name = PackageName::new(name);
let version = Version::from_str(version).map_err(|err| anyhow!(err))?;
let path = path.to_path_buf();
@ -248,7 +249,7 @@ impl InstalledDistribution {
return Ok(None);
};
let name = PackageName::normalize(name);
let name = PackageName::new(name);
let version = Version::from_str(version).map_err(|err| anyhow!(err))?;
let path = path.to_path_buf();
@ -355,7 +356,9 @@ impl<'a> RemoteDistributionRef<'a> {
pub fn id(&self) -> String {
match self {
Self::Registry(name, version, _) => {
format!("{}-{}", DistInfoName::from(*name), version)
// https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory
// `version` is normalized by its `ToString` impl
format!("{}-{}", PackageName::from(*name), version)
}
Self::Url(_name, url) => puffin_cache::digest(&CanonicalUrl::new(url)),
}

1
crates/puffin-installer/Cargo.toml
View file

@ -16,6 +16,7 @@ pep508_rs = { path = "../pep508-rs" }
puffin-client = { path = "../puffin-client" }
puffin-distribution = { path = "../puffin-distribution" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-normalize = { path = "../puffin-normalize" }
puffin-package = { path = "../puffin-package" }
distribution-filename = { path = "../distribution-filename" }

21
crates/puffin-installer/src/plan.rs
View file

@ -6,7 +6,6 @@ use tracing::debug;
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_distribution::{CachedDistribution, InstalledDistribution};
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use crate::url_index::UrlIndex;
use crate::{RegistryIndex, SitePackages};
@ -55,10 +54,8 @@ impl PartitionedRequirements {
let mut extraneous = vec![];
for requirement in requirements {
let package = PackageName::normalize(&requirement.name);
// Filter out already-installed packages.
if let Some(distribution) = site_packages.remove(&package) {
if let Some(distribution) = site_packages.remove(&requirement.name) {
if requirement.is_satisfied_by(distribution.version()) {
debug!("Requirement already satisfied: {distribution}",);
continue;
@ -69,19 +66,21 @@ impl PartitionedRequirements {
// Identify any locally-available distributions that satisfy the requirement.
match requirement.version_or_url.as_ref() {
None | Some(VersionOrUrl::VersionSpecifier(_)) => {
if let Some(distribution) = registry_index.get(&package).filter(|dist| {
let CachedDistribution::Registry(_name, version, _path) = dist else {
return false;
};
requirement.is_satisfied_by(version)
}) {
if let Some(distribution) =
registry_index.get(&requirement.name).filter(|dist| {
let CachedDistribution::Registry(_name, version, _path) = dist else {
return false;
};
requirement.is_satisfied_by(version)
})
{
debug!("Requirement already cached: {distribution}");
local.push(distribution.clone());
continue;
}
}
Some(VersionOrUrl::Url(url)) => {
if let Some(distribution) = url_index.get(&package, url) {
if let Some(distribution) = url_index.get(&requirement.name, url) {
debug!("Requirement already cached: {distribution}");
local.push(distribution.clone());
continue;

2
crates/puffin-installer/src/registry_index.rs
View file

@ -4,7 +4,7 @@ use std::path::Path;
use anyhow::Result;
use puffin_distribution::CachedDistribution;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use crate::cache::{CacheShard, WheelCache};

2
crates/puffin-installer/src/site_packages.rs
View file

@ -5,7 +5,7 @@ use fs_err as fs;
use puffin_distribution::InstalledDistribution;
use puffin_interpreter::Virtualenv;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
#[derive(Debug, Default)]
pub struct SitePackages(BTreeMap<PackageName, InstalledDistribution>);

5
crates/puffin-installer/src/url_index.rs
View file

@ -4,9 +4,10 @@ use anyhow::Result;
use fxhash::FxHashMap;
use url::Url;
use crate::cache::{CacheShard, WheelCache};
use puffin_distribution::{CachedDistribution, RemoteDistributionRef};
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use crate::cache::{CacheShard, WheelCache};
/// A local index of distributions that originate from arbitrary URLs (as opposed to being
/// downloaded from a registry, like `PyPI`).

18
crates/puffin-normalize/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "puffin-normalize"
version = "0.0.1"
edition = "2021"
description = "Normalization for distribution, package and extra anmes"
[dependencies]
anyhow = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
serde = { workspace = true, features = ["derive"] }
[dev-dependencies]
indoc = { version = "2.0.4" }
insta = { version = "1.33.0" }
serde_json = { version = "1.0.107" }
tempfile = { version = "3.8.0" }
test-case = { version = "3.2.1" }

53
crates/puffin-package/src/extra_name.rs → crates/puffin-normalize/src/extra_name.rs
View file

@ -6,6 +6,14 @@ use anyhow::{anyhow, Error, Result};
use once_cell::sync::Lazy;
use regex::Regex;
/// The normalized name of an extra dependency group.
///
/// Converts the name to lowercase and collapses any run of the characters `-`, `_` and `.`
/// down to a single `-`, e.g., `---`, `.`, and `__` all get converted to just `-`.
///
/// See:
/// - <https://peps.python.org/pep-0685/#specification/>
/// - <https://packaging.python.org/en/latest/specifications/name-normalization/>
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ExtraName(String);
@ -19,17 +27,8 @@ static NAME_NORMALIZE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[-_.]+").unwrap()
static NAME_VALIDATE: Lazy<Regex> =
Lazy::new(|| Regex::new(r"(?i)^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$").unwrap());
/// An extra dependency group name.
///
/// See:
/// - <https://peps.python.org/pep-0685/#specification/>
/// - <https://packaging.python.org/en/latest/specifications/name-normalization/>
impl ExtraName {
/// Create a normalized extra name without validation.
///
/// Converts the name to lowercase and collapses any run of the characters `-`, `_` and `.`
/// down to a single `-`, e.g., `---`, `.`, and `__` all get converted to just `-`.
pub fn normalize(name: impl AsRef<str>) -> Self {
pub fn new(name: impl AsRef<str>) -> Self {
// TODO(charlie): Avoid allocating in the common case (when no normalization is required).
let mut normalized = NAME_NORMALIZE.replace_all(name.as_ref(), "-").to_string();
normalized.make_ascii_lowercase();
@ -39,7 +38,7 @@ impl ExtraName {
/// Create a validated, normalized extra name.
pub fn validate(name: impl AsRef<str>) -> Result<Self> {
if NAME_VALIDATE.is_match(name.as_ref()) {
Ok(Self::normalize(name))
Ok(Self::new(name))
} else {
Err(anyhow!(
"Extra names must start and end with a letter or digit and may only contain -, _, ., and alphanumeric characters"
@ -68,32 +67,14 @@ mod tests {
#[test]
fn normalize() {
assert_eq!(ExtraName::new("friendly-bard").as_ref(), "friendly-bard");
assert_eq!(ExtraName::new("Friendly-Bard").as_ref(), "friendly-bard");
assert_eq!(ExtraName::new("FRIENDLY-BARD").as_ref(), "friendly-bard");
assert_eq!(ExtraName::new("friendly.bard").as_ref(), "friendly-bard");
assert_eq!(ExtraName::new("friendly_bard").as_ref(), "friendly-bard");
assert_eq!(ExtraName::new("friendly--bard").as_ref(), "friendly-bard");
assert_eq!(
ExtraName::normalize("friendly-bard").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("Friendly-Bard").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("FRIENDLY-BARD").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("friendly.bard").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("friendly_bard").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("friendly--bard").as_ref(),
"friendly-bard"
);
assert_eq!(
ExtraName::normalize("FrIeNdLy-._.-bArD").as_ref(),
ExtraName::new("FrIeNdLy-._.-bArD").as_ref(),
"friendly-bard"
);
}

5
crates/puffin-normalize/src/lib.rs Normal file
View file

@ -0,0 +1,5 @@
pub use extra_name::ExtraName;
pub use package_name::PackageName;
mod extra_name;
mod package_name;

74
crates/puffin-normalize/src/package_name.rs Normal file
View file

@ -0,0 +1,74 @@
use std::fmt;
use std::fmt::{Display, Formatter};
use once_cell::sync::Lazy;
use regex::Regex;
use serde::{Deserialize, Deserializer, Serialize};
/// The normalized name of a package.
///
/// Converts the name to lowercase and collapses any run of the characters `-`, `_` and `.`
/// down to a single `-`, e.g., `---`, `.`, and `__` all get converted to just `-`.
///
/// See: <https://packaging.python.org/en/latest/specifications/name-normalization/>
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
pub struct PackageName(String);
impl From<&PackageName> for PackageName {
/// Required for `WaitMap::wait`
fn from(package_name: &PackageName) -> Self {
package_name.clone()
}
}
impl Display for PackageName {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
static NAME_NORMALIZE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[-_.]+").unwrap());
impl PackageName {
pub fn new(name: impl AsRef<str>) -> Self {
// TODO(charlie): Avoid allocating in the common case (when no normalization is required).
let mut normalized = NAME_NORMALIZE.replace_all(name.as_ref(), "-").to_string();
normalized.make_ascii_lowercase();
Self(normalized)
}
}
impl AsRef<str> for PackageName {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de> Deserialize<'de> for PackageName {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Ok(Self::new(s))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize() {
assert_eq!(PackageName::new("friendly-bard").as_ref(), "friendly-bard");
assert_eq!(PackageName::new("Friendly-Bard").as_ref(), "friendly-bard");
assert_eq!(PackageName::new("FRIENDLY-BARD").as_ref(), "friendly-bard");
assert_eq!(PackageName::new("friendly.bard").as_ref(), "friendly-bard");
assert_eq!(PackageName::new("friendly_bard").as_ref(), "friendly-bard");
assert_eq!(PackageName::new("friendly--bard").as_ref(), "friendly-bard");
assert_eq!(
PackageName::new("FrIeNdLy-._.-bArD").as_ref(),
"friendly-bard"
);
}
}

2
crates/puffin-package/Cargo.toml
View file

@ -6,11 +6,11 @@ edition = "2021"
[dependencies]
pep440_rs = { path = "../pep440-rs", features = ["serde"] }
pep508_rs = { path = "../pep508-rs", features = ["serde"] }
puffin-normalize = { path = "../puffin-normalize" }
anyhow = { workspace = true }
fs-err = { workspace = true }
mailparse = { workspace = true }
memchr = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
rfc2047-decoder = { workspace = true }

77
crates/puffin-package/src/dist_info_name.rs
View file

@ -1,77 +0,0 @@
use std::fmt;
use std::fmt::{Display, Formatter};
use once_cell::sync::Lazy;
use regex::Regex;
use crate::package_name::PackageName;
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DistInfoName(String);
impl Display for DistInfoName {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
static NAME_NORMALIZE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[-_.]+").unwrap());
impl DistInfoName {
/// See: <https://packaging.python.org/en/latest/specifications/recording-installed-packages/#recording-installed-packages>
pub fn normalize(name: impl AsRef<str>) -> Self {
// TODO(charlie): Avoid allocating in the common case (when no normalization is required).
let mut normalized = NAME_NORMALIZE.replace_all(name.as_ref(), "_").to_string();
normalized.make_ascii_lowercase();
Self(normalized)
}
}
impl AsRef<str> for DistInfoName {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl From<&PackageName> for DistInfoName {
fn from(package_name: &PackageName) -> Self {
Self::normalize(package_name)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize() {
assert_eq!(
DistInfoName::normalize("friendly-bard").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("Friendly-Bard").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("FRIENDLY-BARD").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("friendly.bard").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("friendly_bard").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("friendly--bard").as_ref(),
"friendly_bard"
);
assert_eq!(
DistInfoName::normalize("FrIeNdLy-._.-bArD").as_ref(),
"friendly_bard"
);
}
}

3
crates/puffin-package/src/lib.rs
View file

@ -1,5 +1,2 @@
pub mod dist_info_name;
pub mod extra_name;
pub mod package_name;
pub mod pypi_types;
pub mod requirements_txt;

89
crates/puffin-package/src/package_name.rs
View file

@ -1,89 +0,0 @@
use std::fmt;
use std::fmt::{Display, Formatter};
use once_cell::sync::Lazy;
use regex::Regex;
use crate::dist_info_name::DistInfoName;
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct PackageName(String);
impl From<&PackageName> for PackageName {
/// Required for `WaitMap::wait`
fn from(package_name: &PackageName) -> Self {
package_name.clone()
}
}
impl Display for PackageName {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
static NAME_NORMALIZE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[-_.]+").unwrap());
impl PackageName {
/// Create a normalized representation of a package name.
///
/// Converts the name to lowercase and collapses any run of the characters `-`, `_` and `.`
/// down to a single `-`, e.g., `---`, `.`, and `__` all get converted to just `-`.
///
/// See: <https://packaging.python.org/en/latest/specifications/name-normalization/>
pub fn normalize(name: impl AsRef<str>) -> Self {
// TODO(charlie): Avoid allocating in the common case (when no normalization is required).
let mut normalized = NAME_NORMALIZE.replace_all(name.as_ref(), "-").to_string();
normalized.make_ascii_lowercase();
Self(normalized)
}
}
impl AsRef<str> for PackageName {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl From<DistInfoName> for PackageName {
fn from(dist_info_name: DistInfoName) -> Self {
Self::normalize(dist_info_name)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize() {
assert_eq!(
PackageName::normalize("friendly-bard").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("Friendly-Bard").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("FRIENDLY-BARD").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("friendly.bard").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("friendly_bard").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("friendly--bard").as_ref(),
"friendly-bard"
);
assert_eq!(
PackageName::normalize("FrIeNdLy-._.-bArD").as_ref(),
"friendly-bard"
);
}
}

18
crates/puffin-package/src/pypi_types/metadata.rs
View file

@ -13,6 +13,7 @@ use tracing::warn;
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
use puffin_normalize::PackageName;
/// Python Package Metadata 2.1 as specified in
/// <https://packaging.python.org/specifications/core-metadata/>
@ -24,7 +25,7 @@ use pep508_rs::{Pep508Error, Requirement};
pub struct Metadata21 {
// Mandatory fields
pub metadata_version: String,
pub name: String,
pub name: PackageName,
pub version: Version,
// Optional fields
pub platforms: Vec<String>,
@ -42,7 +43,7 @@ pub struct Metadata21 {
pub license: Option<String>,
pub classifiers: Vec<String>,
pub requires_dist: Vec<Requirement>,
pub provides_dist: Vec<String>,
pub provides_dist: Vec<PackageName>,
pub obsoletes_dist: Vec<String>,
pub requires_python: Option<VersionSpecifiers>,
pub requires_external: Vec<String>,
@ -122,9 +123,11 @@ impl Metadata21 {
let metadata_version = headers
.get_first_value("Metadata-Version")
.ok_or(Error::FieldNotFound("Metadata-Version"))?;
let name = headers
.get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?;
let name = PackageName::new(
headers
.get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?,
);
let version = Version::from_str(
&headers
.get_first_value("Version")
@ -151,7 +154,10 @@ impl Metadata21 {
.iter()
.map(|requires_dist| LenientRequirement::from_str(requires_dist).map(Requirement::from))
.collect::<Result<Vec<_>, _>>()?;
let provides_dist = get_all_values("Provides-Dist");
let provides_dist = get_all_values("Provides-Dist")
.iter()
.map(PackageName::new)
.collect();
let obsoletes_dist = get_all_values("Obsoletes-Dist");
let maintainer = get_first_value("Maintainer");
let maintainer_email = get_first_value("Maintainer-email");

24
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-basic.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -70,7 +74,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "python-dateutil",
name: PackageName(
"python-dateutil",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -102,7 +108,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -133,7 +141,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "six",
name: PackageName(
"six",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -165,7 +175,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "tzdata",
name: PackageName(
"tzdata",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

12
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-a.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "django-debug-toolbar",
name: PackageName(
"django-debug-toolbar",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
],
constraints: [
Requirement {
name: "django",
name: PackageName(
"django",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -66,7 +70,9 @@ RequirementsTxt {
marker: None,
},
Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-b.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "django",
name: PackageName(
"django",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

20
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-for-poetry.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "inflection",
name: PackageName(
"inflection",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "upsidedown",
name: PackageName(
"upsidedown",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -69,7 +73,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: None,
marker: None,
@ -79,10 +85,14 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: Some(
[
"tabulate",
ExtraName(
"tabulate",
),
],
),
version_or_url: Some(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-a.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tomli",
name: PackageName(
"tomli",
),
extras: None,
version_or_url: None,
marker: None,
@ -16,7 +18,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

4
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-b.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tomli",
name: PackageName(
"tomli",
),
extras: None,
version_or_url: None,
marker: None,

20
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-poetry-with-hashes.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "werkzeug",
name: PackageName(
"werkzeug",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -67,7 +69,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "urllib3",
name: PackageName(
"urllib3",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -128,7 +132,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "ansicon",
name: PackageName(
"ansicon",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -200,7 +206,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "requests-oauthlib",
name: PackageName(
"requests-oauthlib",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -262,7 +270,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "psycopg2",
name: PackageName(
"psycopg2",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-small.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tqdm",
name: PackageName(
"tqdm",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "tomli-w",
name: PackageName(
"tomli-w",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

12
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-whitespace.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: None,
marker: None,
@ -16,10 +18,14 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: Some(
[
"tabulate",
ExtraName(
"tabulate",
),
],
),
version_or_url: Some(

24
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-basic.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -70,7 +74,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "python-dateutil",
name: PackageName(
"python-dateutil",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -102,7 +108,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -133,7 +141,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "six",
name: PackageName(
"six",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -165,7 +175,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "tzdata",
name: PackageName(
"tzdata",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

12
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-a.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "django-debug-toolbar",
name: PackageName(
"django-debug-toolbar",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
],
constraints: [
Requirement {
name: "django",
name: PackageName(
"django",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -66,7 +70,9 @@ RequirementsTxt {
marker: None,
},
Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-b.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "django",
name: PackageName(
"django",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pytz",
name: PackageName(
"pytz",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

20
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-for-poetry.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "inflection",
name: PackageName(
"inflection",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "upsidedown",
name: PackageName(
"upsidedown",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -69,7 +73,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: None,
marker: None,
@ -79,10 +85,14 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: Some(
[
"tabulate",
ExtraName(
"tabulate",
),
],
),
version_or_url: Some(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-a.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tomli",
name: PackageName(
"tomli",
),
extras: None,
version_or_url: None,
marker: None,
@ -16,7 +18,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

4
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-b.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tomli",
name: PackageName(
"tomli",
),
extras: None,
version_or_url: None,
marker: None,

20
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-poetry-with-hashes.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "werkzeug",
name: PackageName(
"werkzeug",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -67,7 +69,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "urllib3",
name: PackageName(
"urllib3",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -128,7 +132,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "ansicon",
name: PackageName(
"ansicon",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -200,7 +206,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "requests-oauthlib",
name: PackageName(
"requests-oauthlib",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -262,7 +270,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "psycopg2",
name: PackageName(
"psycopg2",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

8
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-small.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "tqdm",
name: PackageName(
"tqdm",
),
extras: None,
version_or_url: Some(
VersionSpecifier(
@ -38,7 +40,9 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "tomli-w",
name: PackageName(
"tomli-w",
),
extras: None,
version_or_url: Some(
VersionSpecifier(

12
crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-whitespace.txt.snap
View file

@ -6,7 +6,9 @@ RequirementsTxt {
requirements: [
RequirementEntry {
requirement: Requirement {
name: "numpy",
name: PackageName(
"numpy",
),
extras: None,
version_or_url: None,
marker: None,
@ -16,10 +18,14 @@ RequirementsTxt {
},
RequirementEntry {
requirement: Requirement {
name: "pandas",
name: PackageName(
"pandas",
),
extras: Some(
[
"tabulate",
ExtraName(
"tabulate",
),
],
),
version_or_url: Some(

1
crates/puffin-resolver/Cargo.toml
View file

@ -18,6 +18,7 @@ platform-tags = { path = "../platform-tags" }
pubgrub = { path = "../../vendor/pubgrub" }
puffin-client = { path = "../puffin-client" }
puffin-distribution = { path = "../puffin-distribution" }
puffin-normalize = { path = "../puffin-normalize" }
puffin-package = { path = "../puffin-package" }
puffin-traits = { path = "../puffin-traits" }
distribution-filename = { path = "../distribution-filename" }

5
crates/puffin-resolver/src/candidate_selector.rs
View file

@ -2,7 +2,7 @@ use fxhash::FxHashMap;
use pubgrub::range::Range;
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use crate::file::DistributionFile;
use crate::prerelease_mode::PreReleaseStrategy;
@ -59,9 +59,8 @@ impl From<&[Requirement]> for Preferences {
let [version_specifier] = &**version_specifiers else {
return None;
};
let package_name = PackageName::normalize(&requirement.name);
let version = PubGrubVersion::from(version_specifier.version().clone());
Some((package_name, version))
Some((requirement.name.clone(), version))
})
.collect(),
)

6
crates/puffin-resolver/src/prerelease_mode.rs
View file

@ -1,7 +1,7 @@
use fxhash::FxHashSet;
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
@ -70,7 +70,7 @@ impl PreReleaseStrategy {
.iter()
.any(pep440_rs::VersionSpecifier::any_prerelease)
})
.map(|requirement| PackageName::normalize(&requirement.name))
.map(|requirement| requirement.name.clone())
.collect(),
),
PreReleaseMode::IfNecessaryOrExplicit => Self::IfNecessaryOrExplicit(
@ -90,7 +90,7 @@ impl PreReleaseStrategy {
.iter()
.any(pep440_rs::VersionSpecifier::any_prerelease)
})
.map(|requirement| PackageName::normalize(&requirement.name))
.map(|requirement| requirement.name.clone())
.collect(),
),
}

24
crates/puffin-resolver/src/pubgrub/mod.rs
View file

@ -4,8 +4,7 @@ use pubgrub::range::Range;
use tracing::warn;
use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
use puffin_package::dist_info_name::DistInfoName;
use puffin_package::package_name::PackageName;
use puffin_normalize::{ExtraName, PackageName};
pub(crate) use crate::pubgrub::package::PubGrubPackage;
pub(crate) use crate::pubgrub::priority::{PubGrubPriorities, PubGrubPriority};
@ -20,16 +19,15 @@ mod version;
/// Convert a set of requirements to a set of `PubGrub` packages and ranges.
pub(crate) fn iter_requirements<'a>(
requirements: impl Iterator<Item = &'a Requirement> + 'a,
extra: Option<&'a DistInfoName>,
extra: Option<&'a ExtraName>,
source: Option<&'a PackageName>,
env: &'a MarkerEnvironment,
) -> impl Iterator<Item = (PubGrubPackage, Range<PubGrubVersion>)> + 'a {
requirements
.filter(move |requirement| {
let normalized = PackageName::normalize(&requirement.name);
if source.is_some_and(|source| source == &normalized) {
if source.is_some_and(|source| source == &requirement.name) {
// TODO(konstin): Warn only once here
warn!("{normalized} depends on itself");
warn!("{} depends on itself", requirement.name);
false
} else {
true
@ -52,7 +50,7 @@ pub(crate) fn iter_requirements<'a>(
.into_iter()
.flatten()
.map(|extra| {
pubgrub_package(requirement, Some(DistInfoName::normalize(extra))).unwrap()
pubgrub_package(requirement, Some(ExtraName::new(extra))).unwrap()
}),
)
})
@ -79,21 +77,17 @@ pub(crate) fn version_range(specifiers: Option<&VersionOrUrl>) -> Result<Range<P
/// Convert a [`Requirement`] to a `PubGrub`-compatible package and range.
fn pubgrub_package(
requirement: &Requirement,
extra: Option<DistInfoName>,
extra: Option<ExtraName>,
) -> Result<(PubGrubPackage, Range<PubGrubVersion>)> {
match requirement.version_or_url.as_ref() {
// The requirement has no specifier (e.g., `flask`).
None => Ok((
PubGrubPackage::Package(PackageName::normalize(&requirement.name), extra, None),
PubGrubPackage::Package(requirement.name.clone(), extra, None),
Range::full(),
)),
// The requirement has a URL (e.g., `flask @ file:///path/to/flask`).
Some(VersionOrUrl::Url(url)) => Ok((
PubGrubPackage::Package(
PackageName::normalize(&requirement.name),
extra,
Some(url.clone()),
),
PubGrubPackage::Package(requirement.name.clone(), extra, Some(url.clone())),
Range::full(),
)),
// The requirement has a specifier (e.g., `flask>=1.0`).
@ -105,7 +99,7 @@ fn pubgrub_package(
range.intersection(&specifier.into())
})?;
Ok((
PubGrubPackage::Package(PackageName::normalize(&requirement.name), extra, None),
PubGrubPackage::Package(requirement.name.clone(), extra, None),
version,
))
}

5
crates/puffin-resolver/src/pubgrub/package.rs
View file

@ -1,8 +1,7 @@
use derivative::Derivative;
use url::Url;
use puffin_package::dist_info_name::DistInfoName;
use puffin_package::package_name::PackageName;
use puffin_normalize::{ExtraName, PackageName};
/// A PubGrub-compatible wrapper around a "Python package", with two notable characteristics:
///
@ -17,7 +16,7 @@ pub enum PubGrubPackage {
Root,
Package(
PackageName,
Option<DistInfoName>,
Option<ExtraName>,
#[derivative(PartialEq = "ignore")]
#[derivative(PartialOrd = "ignore")]
#[derivative(Hash = "ignore")]

9
crates/puffin-resolver/src/pubgrub/priority.rs
View file

@ -1,8 +1,11 @@
use crate::pubgrub::package::PubGrubPackage;
use fxhash::FxHashMap;
use puffin_package::package_name::PackageName;
use std::cmp::Reverse;
use fxhash::FxHashMap;
use puffin_normalize::PackageName;
use crate::pubgrub::package::PubGrubPackage;
#[derive(Debug, Default)]
pub(crate) struct PubGrubPriorities(FxHashMap<PackageName, usize>);

6
crates/puffin-resolver/src/resolution.rs
View file

@ -10,7 +10,7 @@ use pubgrub::type_aliases::SelectedDependencies;
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{Requirement, VersionOrUrl};
use puffin_distribution::RemoteDistribution;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use puffin_package::pypi_types::File;
use crate::pubgrub::{PubGrubPackage, PubGrubPriority, PubGrubVersion};
@ -139,7 +139,7 @@ impl Graph {
.node_indices()
.map(|node| match &self.0[node] {
RemoteDistribution::Registry(name, version, _file) => Requirement {
name: name.to_string(),
name: name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from(
VersionSpecifier::equals_version(version.clone()),
@ -147,7 +147,7 @@ impl Graph {
marker: None,
},
RemoteDistribution::Url(name, url) => Requirement {
name: name.to_string(),
name: name.clone(),
extras: None,
version_or_url: Some(VersionOrUrl::Url(url.clone())),
marker: None,

4
crates/puffin-resolver/src/resolution_mode.rs
View file

@ -1,7 +1,7 @@
use fxhash::FxHashSet;
use pep508_rs::Requirement;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
@ -37,7 +37,7 @@ impl ResolutionStrategy {
ResolutionMode::LowestDirect => Self::LowestDirect(
direct_dependencies
.iter()
.map(|requirement| PackageName::normalize(&requirement.name))
.map(|requirement| requirement.name.clone())
.collect(),
),
}

15
crates/puffin-resolver/src/resolver.rs
View file

@ -23,8 +23,7 @@ use pep508_rs::{MarkerEnvironment, Requirement};
use platform_tags::Tags;
use puffin_client::RegistryClient;
use puffin_distribution::{RemoteDistributionRef, VersionOrUrl};
use puffin_package::dist_info_name::DistInfoName;
use puffin_package::package_name::PackageName;
use puffin_normalize::{ExtraName, PackageName};
use puffin_package::pypi_types::{File, Metadata21, SimpleJson};
use puffin_traits::BuildContext;
@ -126,7 +125,7 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> {
// Push all the requirements into the package sink.
for requirement in &self.requirements {
debug!("Adding root dependency: {requirement}");
let package_name = PackageName::normalize(&requirement.name);
let package_name = requirement.name.clone();
match &requirement.version_or_url {
// If this is a registry-based package, fetch the package metadata.
None | Some(pep508_rs::VersionOrUrl::VersionSpecifier(_)) => {
@ -535,11 +534,7 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> {
// If any packages were further constrained by the user, add those constraints.
for constraint in &self.constraints {
let package = PubGrubPackage::Package(
PackageName::normalize(&constraint.name),
None,
None,
);
let package = PubGrubPackage::Package(constraint.name.clone(), None, None);
if let Some(range) = constraints.get_mut(&package) {
*range = range.intersection(
&version_range(constraint.version_or_url.as_ref()).unwrap(),
@ -551,7 +546,7 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> {
if !metadata
.provides_extras
.iter()
.any(|provided_extra| DistInfoName::normalize(provided_extra) == *extra)
.any(|provided_extra| ExtraName::new(provided_extra) == *extra)
{
return Ok(Dependencies::Unknown);
}
@ -773,7 +768,7 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> {
pub trait Reporter: Send + Sync {
/// Callback to invoke when a dependency is resolved.
fn on_progress(&self, name: &PackageName, extra: Option<&DistInfoName>, version: VersionOrUrl);
fn on_progress(&self, name: &PackageName, extra: Option<&ExtraName>, version: VersionOrUrl);
/// Callback to invoke when the resolution is complete.
fn on_complete(&self);

8
crates/puffin-resolver/src/wheel_finder.rs
View file

@ -16,7 +16,7 @@ use pep508_rs::{Requirement, VersionOrUrl};
use platform_tags::Tags;
use puffin_client::RegistryClient;
use puffin_distribution::RemoteDistribution;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use puffin_package::pypi_types::{File, Metadata21, SimpleJson};
use crate::error::ResolveError;
@ -85,7 +85,7 @@ impl<'a> WheelFinder<'a> {
package_sink.unbounded_send(Request::Package(requirement.clone()))?;
}
Some(VersionOrUrl::Url(url)) => {
let package_name = PackageName::normalize(&requirement.name);
let package_name = requirement.name.clone();
let package = RemoteDistribution::from_url(package_name.clone(), url.clone());
resolution.insert(package_name, package);
}
@ -131,7 +131,7 @@ impl<'a> WheelFinder<'a> {
);
let package = RemoteDistribution::from_registry(
PackageName::normalize(&metadata.name),
metadata.name,
metadata.version,
file,
);
@ -141,7 +141,7 @@ impl<'a> WheelFinder<'a> {
}
// Add to the resolved set.
let normalized_name = PackageName::normalize(&requirement.name);
let normalized_name = requirement.name.clone();
resolution.insert(normalized_name, package);
}
}

2
crates/puffin-workspace/Cargo.toml
View file

@ -12,7 +12,7 @@ license = { workspace = true }
[dependencies]
pep440_rs = { path = "../pep440-rs" }
pep508_rs = { path = "../pep508-rs" }
puffin-package = { path = "../puffin-package" }
puffin-normalize = { path = "../puffin-normalize" }
fs-err = { workspace = true }
pyproject-toml = { workspace = true }

7
crates/puffin-workspace/src/workspace.rs
View file

@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
use toml_edit::Document;
use pep508_rs::Requirement;
use puffin_package::package_name::PackageName;
use puffin_normalize::PackageName;
use crate::toml::format_multiline_array;
use crate::verbatim::VerbatimRequirement;
@ -85,8 +85,7 @@ impl Workspace {
return false;
};
PackageName::normalize(&requirement.requirement.name)
== PackageName::normalize(existing.name)
requirement.requirement.name == existing.name
});
if let Some(index) = index {
@ -124,7 +123,7 @@ impl Workspace {
return false;
};
PackageName::normalize(name) == PackageName::normalize(existing.name)
PackageName::new(name) == existing.name
});
let Some(index) = index else {