mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-10-27 10:26:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Build backend: Include readme and license files (#9149)

Browse source 
When building source distributions, we need to include the readme, so it
can become part the METADATA body when building the wheel. We also need
to support the license files from PEP 639. When building the source
distribution, we copy those file relative to their origin, when building
the wheel, we copy them to `.dist-info/licenses`.

The test for idempotence in wheel building is merged into the file
listing test, which also covers that source tree -> source dist -> wheel
is equivalent to source tree -> wheel, though we do need to check for
file inclusion stronger here.

Best reviewed commit-by-commit
This commit is contained in:
konsti 2024-11-15 15:41:39 +01:00 • committed by GitHub
parent dafbd7d405
commit 21d570fac9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 509 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

136
crates/uv-build-backend/src/lib.rs
View file

@ -4,14 +4,14 @@ use crate::metadata::{PyProjectToml, ValidationError};
use flate2::write::GzEncoder;
use flate2::Compression;
use fs_err::File;
use globset::GlobSetBuilder;
use globset::{Glob, GlobSetBuilder};
use itertools::Itertools;
use sha2::{Digest, Sha256};
use std::fs::FileType;
use std::io::{BufReader, Cursor, Read, Write};
use std::path::{Path, PathBuf, StripPrefixError};
use std::{io, mem};
use tar::{Builder, EntryType, Header};
use tar::{EntryType, Header};
use thiserror::Error;
use tracing::{debug, trace};
use uv_distribution_filename::{SourceDistExtension, SourceDistFilename, WheelFilename};
@ -54,8 +54,6 @@ pub enum Error {
#[source]
err: walkdir::Error,
},
#[error("Non-UTF-8 paths are not supported: `{}`", _0.user_display())]
NotUtf8Path(PathBuf),
#[error("Failed to walk source tree")]
StripPrefix(#[from] StripPrefixError),
#[error("Unsupported file type {1:?}: `{}`", _0.user_display())]
@ -356,17 +354,16 @@ pub fn build_wheel(
let relative_path = entry
.path()
.strip_prefix(&strip_root)
.expect("walkdir starts with root");
let relative_path_str = relative_path
.to_str()
.ok_or_else(|| Error::NotUtf8Path(relative_path.to_path_buf()))?;
.expect("walkdir starts with root")
.user_display()
.to_string();
debug!("Adding to wheel: `{relative_path_str}`");
debug!("Adding to wheel: `{relative_path}`");
if entry.file_type().is_dir() {
wheel_writer.write_directory(relative_path_str)?;
wheel_writer.write_directory(&relative_path)?;
} else if entry.file_type().is_file() {
wheel_writer.write_file(relative_path_str, entry.path())?;
wheel_writer.write_file(&relative_path, entry.path())?;
} else {
// TODO(konsti): We may want to support symlinks, there is support for installing them.
return Err(Error::UnsupportedFileType(
@ -378,7 +375,80 @@ pub fn build_wheel(
entry.path();
}
debug!("Adding metadata files to {}", wheel_path.user_display());
if let Some(license_files) = &pyproject_toml.license_files() {
let license_files_globs: Vec<_> = license_files
.iter()
.map(|license_files| {
trace!("Including license files at: `{license_files}`");
parse_portable_glob(license_files)
})
.collect::<Result<_, _>>()
.map_err(|err| Error::PortableGlob {
field: "project.license-files".to_string(),
source: err,
})?;
let license_files_matcher =
GlobDirFilter::from_globs(&license_files_globs).map_err(|err| {
Error::GlobSetTooLarge {
field: "project.license-files".to_string(),
source: err,
}
})?;
let license_dir = format!(
"{}-{}.dist-info/licenses/",
pyproject_toml.name().as_dist_info_name(),
pyproject_toml.version()
);
wheel_writer.write_directory(&license_dir)?;
for entry in WalkDir::new(source_tree).into_iter().filter_entry(|entry| {
// TODO(konsti): This should be prettier.
let relative = entry
.path()
.strip_prefix(source_tree)
.expect("walkdir starts with root");
// Fast path: Don't descend into a directory that can't be included.
license_files_matcher.match_directory(relative)
}) {
let entry = entry.map_err(|err| Error::WalkDir {
root: source_tree.to_path_buf(),
err,
})?;
// TODO(konsti): This should be prettier.
let relative = entry
.path()
.strip_prefix(source_tree)
.expect("walkdir starts with root");
if !license_files_matcher.match_path(relative) {
trace!("Excluding {}", relative.user_display());
continue;
};
let relative_licenses = Path::new(&license_dir)
.join(relative)
.portable_display()
.to_string();
if entry.file_type().is_dir() {
wheel_writer.write_directory(&relative_licenses)?;
} else if entry.file_type().is_file() {
debug!("Adding license file: `{}`", relative.user_display());
wheel_writer.write_file(&relative_licenses, entry.path())?;
} else {
// TODO(konsti): We may want to support symlinks, there is support for installing them.
return Err(Error::UnsupportedFileType(
entry.path().to_path_buf(),
entry.file_type(),
));
}
}
}
debug!("Adding metadata files to: `{}`", wheel_path.user_display());
let dist_info_dir = write_dist_info(
&mut wheel_writer,
&pyproject_toml,
@ -449,28 +519,32 @@ pub fn build_source_dist(
extension: SourceDistExtension::TarGz,
};
let top_level = format!("{}-{}", pyproject_toml.name(), pyproject_toml.version());
let top_level = format!(
"{}-{}",
pyproject_toml.name().as_dist_info_name(),
pyproject_toml.version()
);
let source_dist_path = source_dist_directory.join(filename.to_string());
let tar_gz = File::create(&source_dist_path)?;
let enc = GzEncoder::new(tar_gz, Compression::default());
let mut tar = tar::Builder::new(enc);
let metadata = pyproject_toml
.to_metadata(source_tree)?
.core_metadata_format();
let metadata = pyproject_toml.to_metadata(source_tree)?;
let metadata_email = metadata.core_metadata_format();
let mut header = Header::new_gnu();
header.set_size(metadata.bytes().len() as u64);
header.set_size(metadata_email.bytes().len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar.append_data(
&mut header,
Path::new(&top_level).join("PKG-INFO"),
Cursor::new(metadata),
Cursor::new(metadata_email),
)
.map_err(|err| Error::TarWrite(source_dist_path.clone(), err))?;
// The user (or default) includes
let mut include_globs = Vec::new();
for include in settings.include {
let glob = parse_portable_glob(&include).map_err(|err| Error::PortableGlob {
@ -479,6 +553,30 @@ pub fn build_source_dist(
})?;
include_globs.push(glob.clone());
}
// Include the Readme
if let Some(readme) = pyproject_toml
.readme()
.as_ref()
.and_then(|readme| readme.path())
{
trace!("Including readme at: `{}`", readme.user_display());
include_globs.push(
Glob::new(&globset::escape(&readme.portable_display().to_string()))
.expect("escaped globset is parseable"),
);
}
// Include the license files
for license_files in pyproject_toml.license_files().into_iter().flatten() {
trace!("Including license files at: `{license_files}`");
let glob = parse_portable_glob(license_files).map_err(|err| Error::PortableGlob {
field: "project.license-files".to_string(),
source: err,
})?;
include_globs.push(glob);
}
let include_matcher =
GlobDirFilter::from_globs(&include_globs).map_err(|err| Error::GlobSetTooLarge {
field: "tool.uv.source-dist.include".to_string(),
@ -549,7 +647,7 @@ pub fn build_source_dist(
/// Add a file or a directory to a source distribution.
fn add_source_dist_entry(
tar: &mut Builder<GzEncoder<File>>,
tar: &mut tar::Builder<GzEncoder<File>>,
entry: &DirEntry,
top_level: &str,
source_dist_path: &Path,

25
crates/uv-build-backend/src/metadata.rs
View file

@ -76,6 +76,14 @@ impl PyProjectToml {
Ok(toml::from_str(contents)?)
}
pub(crate) fn readme(&self) -> Option<&Readme> {
self.project.readme.as_ref()
}
pub(crate) fn license_files(&self) -> Option<&[String]> {
self.project.license_files.as_deref()
}
/// Warn if the `[build-system]` table looks suspicious.
///
/// Example of a valid table:
@ -591,7 +599,7 @@ struct Project {
/// <https://packaging.python.org/en/latest/specifications/pyproject-toml/#readme>.
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged, rename_all = "kebab-case")]
enum Readme {
pub(crate) enum Readme {
/// Relative path to the README.
String(PathBuf),
/// Relative path to the README.
@ -608,11 +616,22 @@ enum Readme {
},
}
impl Readme {
/// If the readme is a file, return the path to the file.
pub(crate) fn path(&self) -> Option<&Path> {
match self {
Readme::String(path) => Some(path),
Readme::File { file, .. } => Some(file),
Readme::Text { .. } => None,
}
}
}
/// The optional `project.license` key in a pyproject.toml as specified in
/// <https://packaging.python.org/en/latest/specifications/pyproject-toml/#license>.
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged)]
enum License {
pub(crate) enum License {
/// An SPDX Expression.
///
/// From the provisional PEP 639.
@ -639,7 +658,7 @@ enum License {
deny_unknown_fields,
expecting = "a table with 'name' and/or 'email' keys"
)]
enum Contact {
pub(crate) enum Contact {
/// TODO(konsti): RFC 822 validation.
NameEmail { name: String, email: String },
/// TODO(konsti): RFC 822 validation.

194
crates/uv-build-backend/src/tests.rs
View file

@ -1,5 +1,5 @@
use super::*;
use indoc::indoc;
use flate2::bufread::GzDecoder;
use insta::assert_snapshot;
use std::str::FromStr;
use tempfile::TempDir;
@ -43,56 +43,6 @@ fn test_record() {
");
}
/// Check that we write deterministic wheels.
#[test]
fn test_determinism() {
let built_by_uv = Path::new("../../scripts/packages/built-by-uv");
let src = TempDir::new().unwrap();
for dir in ["src", "tests", "data-dir"] {
copy_dir_all(built_by_uv.join(dir), src.path().join(dir)).unwrap();
}
for dir in ["pyproject.toml", "README.md", "uv.lock"] {
fs_err::copy(built_by_uv.join(dir), src.path().join(dir)).unwrap();
}
let temp1 = TempDir::new().unwrap();
build_wheel(
src.path(),
temp1.path(),
None,
WheelSettings::default(),
"1.0.0+test",
)
.unwrap();
// Touch the file to check that we don't serialize the last modified date.
fs_err::write(
src.path().join("src/built_by_uv/__init__.py"),
indoc! {r#"
def greet() -> str:
return "Hello 👋"
"#
},
)
.unwrap();
let temp2 = TempDir::new().unwrap();
build_wheel(
src.path(),
temp2.path(),
None,
WheelSettings::default(),
"1.0.0+test",
)
.unwrap();
let wheel_filename = "built_by_uv-0.1.0-py3-none-any.whl";
assert_eq!(
fs_err::read(temp1.path().join(wheel_filename)).unwrap(),
fs_err::read(temp2.path().join(wheel_filename)).unwrap()
);
}
/// Snapshot all files from the prepare metadata hook.
#[test]
fn test_prepare_metadata() {
@ -125,12 +75,17 @@ fn test_prepare_metadata() {
.path()
.join("built_by_uv-0.1.0.dist-info/METADATA");
assert_snapshot!(fs_err::read_to_string(metadata_file).unwrap(), @r###"
Metadata-Version: 2.3
Metadata-Version: 2.4
Name: built-by-uv
Version: 0.1.0
Summary: A package to be built with the uv build backend that uses all features exposed by the build backend
Requires-Dist: anyio>=4,<5
Requires-Python: >=3.12
Description-Content-Type: text/markdown
# built_by_uv
A package to be built with the uv build backend that uses all features exposed by the build backend.
"###);
let record_file = metadata_dir
@ -138,7 +93,7 @@ fn test_prepare_metadata() {
.join("built_by_uv-0.1.0.dist-info/RECORD");
assert_snapshot!(fs_err::read_to_string(record_file).unwrap(), @r###"
built_by_uv-0.1.0.dist-info/WHEEL,sha256=3da1bfa0e8fd1b6cc246aa0b2b44a35815596c600cb485c39a6f8c106c3d5a8d,83
built_by_uv-0.1.0.dist-info/METADATA,sha256=ec36b5ae8830bdd248e90aaf581483ffb057f9a2d0f41e19e585531e7d07c9dc,215
built_by_uv-0.1.0.dist-info/METADATA,sha256=acb91f5a18cb53fa57b45eb4590ea13195a774c856a9dd8cf27cc5435d6451b6,372
built_by_uv-0.1.0.dist-info/RECORD,,
"###);
@ -152,3 +107,136 @@ fn test_prepare_metadata() {
Tag: py3-none-any
"###);
}
/// Test that source tree -> source dist -> wheel includes the right files and is stable and
/// deterministic in dependent of the build path.
#[test]
fn built_by_uv_building() {
let built_by_uv = Path::new("../../scripts/packages/built-by-uv");
let src = TempDir::new().unwrap();
for dir in ["src", "tests", "data-dir", "third-party-licenses"] {
copy_dir_all(built_by_uv.join(dir), src.path().join(dir)).unwrap();
}
for dir in [
"pyproject.toml",
"README.md",
"uv.lock",
"LICENSE-APACHE",
"LICENSE-MIT",
] {
fs_err::copy(built_by_uv.join(dir), src.path().join(dir)).unwrap();
}
// Build a wheel from the source tree
let direct_output_dir = TempDir::new().unwrap();
build_wheel(
src.path(),
direct_output_dir.path(),
None,
WheelSettings::default(),
"1.0.0+test",
)
.unwrap();
let wheel = zip::ZipArchive::new(
File::open(
direct_output_dir
.path()
.join("built_by_uv-0.1.0-py3-none-any.whl"),
)
.unwrap(),
)
.unwrap();
let mut direct_wheel_contents: Vec<_> = wheel.file_names().collect();
direct_wheel_contents.sort_unstable();
// Build a source dist from the source tree
let source_dist_dir = TempDir::new().unwrap();
build_source_dist(
src.path(),
source_dist_dir.path(),
SourceDistSettings::default(),
"1.0.0+test",
)
.unwrap();
// Build a wheel from the source dist
let sdist_tree = TempDir::new().unwrap();
let source_dist_path = source_dist_dir.path().join("built_by_uv-0.1.0.tar.gz");
let sdist_reader = BufReader::new(File::open(&source_dist_path).unwrap());
let mut source_dist = tar::Archive::new(GzDecoder::new(sdist_reader));
let mut source_dist_contents: Vec<_> = source_dist
.entries()
.unwrap()
.map(|entry| entry.unwrap().path().unwrap().to_str().unwrap().to_string())
.collect();
source_dist_contents.sort();
// Reset the reader and unpack
let sdist_reader = BufReader::new(File::open(&source_dist_path).unwrap());
let mut source_dist = tar::Archive::new(GzDecoder::new(sdist_reader));
source_dist.unpack(sdist_tree.path()).unwrap();
drop(source_dist_dir);
let indirect_output_dir = TempDir::new().unwrap();
build_wheel(
&sdist_tree.path().join("built_by_uv-0.1.0"),
indirect_output_dir.path(),
None,
WheelSettings::default(),
"1.0.0+test",
)
.unwrap();
// Check that we write deterministic wheels.
let wheel_filename = "built_by_uv-0.1.0-py3-none-any.whl";
assert_eq!(
fs_err::read(direct_output_dir.path().join(wheel_filename)).unwrap(),
fs_err::read(indirect_output_dir.path().join(wheel_filename)).unwrap()
);
// Check the contained files and directories
assert_snapshot!(source_dist_contents.iter().map(|path| path.replace('\\', "/")).join("\n"), @r"
built_by_uv-0.1.0/LICENSE-APACHE
built_by_uv-0.1.0/LICENSE-MIT
built_by_uv-0.1.0/PKG-INFO
built_by_uv-0.1.0/README.md
built_by_uv-0.1.0/pyproject.toml
built_by_uv-0.1.0/src/built_by_uv
built_by_uv-0.1.0/src/built_by_uv/__init__.py
built_by_uv-0.1.0/src/built_by_uv/arithmetic
built_by_uv-0.1.0/src/built_by_uv/arithmetic/__init__.py
built_by_uv-0.1.0/src/built_by_uv/arithmetic/circle.py
built_by_uv-0.1.0/src/built_by_uv/arithmetic/pi.txt
built_by_uv-0.1.0/third-party-licenses/PEP-401.txt
");
let wheel = zip::ZipArchive::new(
File::open(
indirect_output_dir
.path()
.join("built_by_uv-0.1.0-py3-none-any.whl"),
)
.unwrap(),
)
.unwrap();
let mut indirect_wheel_contents: Vec<_> = wheel.file_names().collect();
indirect_wheel_contents.sort_unstable();
assert_eq!(indirect_wheel_contents, direct_wheel_contents);
assert_snapshot!(indirect_wheel_contents.iter().map(|path| path.replace('\\', "/")).join("\n"), @r"
built_by_uv-0.1.0.dist-info/
built_by_uv-0.1.0.dist-info/METADATA
built_by_uv-0.1.0.dist-info/RECORD
built_by_uv-0.1.0.dist-info/WHEEL
built_by_uv-0.1.0.dist-info/licenses/
built_by_uv-0.1.0.dist-info/licenses/LICENSE-APACHE
built_by_uv-0.1.0.dist-info/licenses/LICENSE-MIT
built_by_uv-0.1.0.dist-info/licenses/third-party-licenses/PEP-401.txt
built_by_uv/
built_by_uv/__init__.py
built_by_uv/arithmetic/
built_by_uv/arithmetic/__init__.py
built_by_uv/arithmetic/circle.py
built_by_uv/arithmetic/pi.txt
");
}