From fee6ab58c0a98fbd89c9f8a3fc083a5e75fb4948 Mon Sep 17 00:00:00 2001 From: konsti Date: Tue, 3 Dec 2024 11:58:02 +0100 Subject: [PATCH] Build backend: Add functions to collect file list (#9602) Using the directory writer trait, we can collect the files instead of writing them to a real sink. This builds up to a `uv build --list` similar to `cargo package --list`. It is not connected to the cli yet. --- crates/uv-build-backend/src/lib.rs | 98 ++++++++++++++++++++-- crates/uv-build-backend/src/source_dist.rs | 22 ++++- crates/uv-build-backend/src/wheel.rs | 72 +++++++++++++--- 3 files changed, 173 insertions(+), 19 deletions(-) diff --git a/crates/uv-build-backend/src/lib.rs b/crates/uv-build-backend/src/lib.rs index 9d926d50d..1a2779434 100644 --- a/crates/uv-build-backend/src/lib.rs +++ b/crates/uv-build-backend/src/lib.rs @@ -3,8 +3,8 @@ mod source_dist; mod wheel; pub use metadata::PyProjectToml; -pub use source_dist::build_source_dist; -pub use wheel::{build_editable, build_wheel, metadata}; +pub use source_dist::{build_source_dist, list_source_dist}; +pub use wheel::{build_editable, build_wheel, list_wheel, metadata}; use crate::metadata::ValidationError; use std::fs::FileType; @@ -77,6 +77,8 @@ pub enum Error { /// error case). trait DirectoryWriter { /// Add a file with the given content. + /// + /// Files added through the method are considered generated when listing included files. fn write_bytes(&mut self, path: &str, bytes: &[u8]) -> Result<(), Error>; /// Add a local file. @@ -89,6 +91,42 @@ trait DirectoryWriter { fn close(self, dist_info_dir: &str) -> Result<(), Error>; } +/// Name of the file in the archive and path outside, if it wasn't generated. +pub(crate) type FileList = Vec<(String, Option)>; + +/// A dummy writer to collect the file names that would be included in a build. +pub(crate) struct ListWriter<'a> { + files: &'a mut FileList, +} + +impl<'a> ListWriter<'a> { + /// Convert the writer to the collected file names. + pub(crate) fn new(files: &'a mut FileList) -> Self { + Self { files } + } +} + +impl DirectoryWriter for ListWriter<'_> { + fn write_bytes(&mut self, path: &str, _bytes: &[u8]) -> Result<(), Error> { + self.files.push((path.to_string(), None)); + Ok(()) + } + + fn write_file(&mut self, path: &str, file: &Path) -> Result<(), Error> { + self.files + .push((path.to_string(), Some(file.to_path_buf()))); + Ok(()) + } + + fn write_directory(&mut self, _directory: &str) -> Result<(), Error> { + Ok(()) + } + + fn close(self, _dist_info_dir: &str) -> Result<(), Error> { + Ok(()) + } +} + /// PEP 517 requires that the metadata directory from the prepare metadata call is identical to the /// build wheel call. This method performs a prudence check that `METADATA` and `entry_points.txt` /// match. @@ -140,14 +178,13 @@ fn check_metadata_directory( #[cfg(test)] mod tests { use super::*; - use crate::source_dist::build_source_dist; use flate2::bufread::GzDecoder; use fs_err::File; use insta::assert_snapshot; use itertools::Itertools; use std::io::BufReader; use tempfile::TempDir; - use uv_fs::copy_dir_all; + use uv_fs::{copy_dir_all, relative_to}; /// Test that source tree -> source dist -> wheel includes the right files and is stable and /// deterministic in dependent of the build path. @@ -184,6 +221,7 @@ mod tests { // Build a wheel from the source tree let direct_output_dir = TempDir::new().unwrap(); + let (_name, wheel_list_files) = list_wheel(src.path(), "1.0.0+test").unwrap(); build_wheel(src.path(), direct_output_dir.path(), None, "1.0.0+test").unwrap(); let wheel = zip::ZipArchive::new( @@ -198,8 +236,9 @@ mod tests { let mut direct_wheel_contents: Vec<_> = wheel.file_names().collect(); direct_wheel_contents.sort_unstable(); - // Build a source dist from the source tree + // List file and build a source dist from the source tree let source_dist_dir = TempDir::new().unwrap(); + let (_name, source_dist_list_files) = list_source_dist(src.path(), "1.0.0+test").unwrap(); build_source_dist(src.path(), source_dist_dir.path(), "1.0.0+test").unwrap(); // Build a wheel from the source dist @@ -240,6 +279,24 @@ mod tests { indirect_wheel_contents.sort_unstable(); assert_eq!(indirect_wheel_contents, direct_wheel_contents); + let format_file_list = |file_list: FileList| { + file_list + .into_iter() + .map(|(path, source)| { + let path = path.replace('\\', "/"); + if let Some(source) = source { + let source = relative_to(source, src.path()) + .unwrap() + .portable_display() + .to_string(); + format!("{path} ({source})") + } else { + format!("{path} (generated)") + } + }) + .join("\n") + }; + // Check the contained files and directories assert_snapshot!(source_dist_contents.iter().map(|path| path.replace('\\', "/")).join("\n"), @r###" built_by_uv-0.1.0/ @@ -265,6 +322,22 @@ mod tests { built_by_uv-0.1.0/third-party-licenses built_by_uv-0.1.0/third-party-licenses/PEP-401.txt "###); + assert_snapshot!(format_file_list(source_dist_list_files), @r###" + built_by_uv-0.1.0/LICENSE-APACHE (LICENSE-APACHE) + built_by_uv-0.1.0/LICENSE-MIT (LICENSE-MIT) + built_by_uv-0.1.0/PKG-INFO (generated) + built_by_uv-0.1.0/README.md (README.md) + built_by_uv-0.1.0/assets/data.csv (assets/data.csv) + built_by_uv-0.1.0/header/built_by_uv.h (header/built_by_uv.h) + built_by_uv-0.1.0/pyproject.toml (pyproject.toml) + built_by_uv-0.1.0/scripts/whoami.sh (scripts/whoami.sh) + built_by_uv-0.1.0/src/built_by_uv/__init__.py (src/built_by_uv/__init__.py) + built_by_uv-0.1.0/src/built_by_uv/arithmetic/__init__.py (src/built_by_uv/arithmetic/__init__.py) + built_by_uv-0.1.0/src/built_by_uv/arithmetic/circle.py (src/built_by_uv/arithmetic/circle.py) + built_by_uv-0.1.0/src/built_by_uv/arithmetic/pi.txt (src/built_by_uv/arithmetic/pi.txt) + built_by_uv-0.1.0/src/built_by_uv/build-only.h (src/built_by_uv/build-only.h) + built_by_uv-0.1.0/third-party-licenses/PEP-401.txt (third-party-licenses/PEP-401.txt) + "###); assert_snapshot!(indirect_wheel_contents.iter().map(|path| path.replace('\\', "/")).join("\n"), @r###" built_by_uv-0.1.0.data/data/ @@ -290,6 +363,21 @@ mod tests { built_by_uv/arithmetic/pi.txt "###); + assert_snapshot!(format_file_list(wheel_list_files), @r###" + built_by_uv-0.1.0.data/data/data.csv (assets/data.csv) + built_by_uv-0.1.0.data/headers/built_by_uv.h (header/built_by_uv.h) + built_by_uv-0.1.0.data/scripts/whoami.sh (scripts/whoami.sh) + built_by_uv-0.1.0.dist-info/METADATA (generated) + built_by_uv-0.1.0.dist-info/WHEEL (generated) + built_by_uv-0.1.0.dist-info/licenses/LICENSE-APACHE (LICENSE-APACHE) + built_by_uv-0.1.0.dist-info/licenses/LICENSE-MIT (LICENSE-MIT) + built_by_uv-0.1.0.dist-info/licenses/third-party-licenses/PEP-401.txt (third-party-licenses/PEP-401.txt) + built_by_uv/__init__.py (src/built_by_uv/__init__.py) + built_by_uv/arithmetic/__init__.py (src/built_by_uv/arithmetic/__init__.py) + built_by_uv/arithmetic/circle.py (src/built_by_uv/arithmetic/circle.py) + built_by_uv/arithmetic/pi.txt (src/built_by_uv/arithmetic/pi.txt) + "###); + // Check that we write deterministic wheels. let wheel_filename = "built_by_uv-0.1.0-py3-none-any.whl"; assert_eq!( diff --git a/crates/uv-build-backend/src/source_dist.rs b/crates/uv-build-backend/src/source_dist.rs index 72bc4d887..e93260581 100644 --- a/crates/uv-build-backend/src/source_dist.rs +++ b/crates/uv-build-backend/src/source_dist.rs @@ -1,6 +1,6 @@ use crate::metadata::{BuildBackendSettings, DEFAULT_EXCLUDES}; use crate::wheel::build_exclude_matcher; -use crate::{DirectoryWriter, Error, PyProjectToml}; +use crate::{DirectoryWriter, Error, FileList, ListWriter, PyProjectToml}; use flate2::write::GzEncoder; use flate2::Compression; use fs_err::File; @@ -35,6 +35,26 @@ pub fn build_source_dist( Ok(filename) } +/// List the files that would be included in a source distribution and their origin. +pub fn list_source_dist( + source_tree: &Path, + uv_version: &str, +) -> Result<(SourceDistFilename, FileList), Error> { + let contents = fs_err::read_to_string(source_tree.join("pyproject.toml"))?; + let pyproject_toml = PyProjectToml::parse(&contents)?; + let filename = SourceDistFilename { + name: pyproject_toml.name().clone(), + version: pyproject_toml.version().clone(), + extension: SourceDistExtension::TarGz, + }; + let mut files = FileList::new(); + let writer = ListWriter::new(&mut files); + write_source_dist(source_tree, writer, uv_version)?; + // Ensure a deterministic order even when file walking changes + files.sort_unstable(); + Ok((filename, files)) +} + /// Build includes and excludes for source tree walking for source dists. fn source_dist_matcher( pyproject_toml: &PyProjectToml, diff --git a/crates/uv-build-backend/src/wheel.rs b/crates/uv-build-backend/src/wheel.rs index e1be71a93..d4d4ba712 100644 --- a/crates/uv-build-backend/src/wheel.rs +++ b/crates/uv-build-backend/src/wheel.rs @@ -1,5 +1,5 @@ use crate::metadata::{BuildBackendSettings, DEFAULT_EXCLUDES}; -use crate::{DirectoryWriter, Error, PyProjectToml}; +use crate::{DirectoryWriter, Error, FileList, ListWriter, PyProjectToml}; use fs_err::File; use globset::{GlobSet, GlobSetBuilder}; use itertools::Itertools; @@ -27,11 +27,6 @@ pub fn build_wheel( for warning in pyproject_toml.check_build_system(uv_version) { warn_user_once!("{warning}"); } - let settings = pyproject_toml - .settings() - .cloned() - .unwrap_or_else(BuildBackendSettings::default); - crate::check_metadata_directory(source_tree, metadata_directory, &pyproject_toml)?; let filename = WheelFilename { @@ -45,7 +40,58 @@ pub fn build_wheel( let wheel_path = wheel_dir.join(filename.to_string()); debug!("Writing wheel at {}", wheel_path.user_display()); - let mut wheel_writer = ZipDirectoryWriter::new_wheel(File::create(&wheel_path)?); + let wheel_writer = ZipDirectoryWriter::new_wheel(File::create(&wheel_path)?); + + write_wheel( + source_tree, + &pyproject_toml, + &filename, + uv_version, + wheel_writer, + )?; + + Ok(filename) +} + +/// List the files that would be included in a source distribution and their origin. +pub fn list_wheel( + source_tree: &Path, + uv_version: &str, +) -> Result<(WheelFilename, FileList), Error> { + let contents = fs_err::read_to_string(source_tree.join("pyproject.toml"))?; + let pyproject_toml = PyProjectToml::parse(&contents)?; + for warning in pyproject_toml.check_build_system(uv_version) { + warn_user_once!("{warning}"); + } + + let filename = WheelFilename { + name: pyproject_toml.name().clone(), + version: pyproject_toml.version().clone(), + build_tag: None, + python_tag: vec!["py3".to_string()], + abi_tag: vec!["none".to_string()], + platform_tag: vec!["any".to_string()], + }; + + let mut files = FileList::new(); + let writer = ListWriter::new(&mut files); + write_wheel(source_tree, &pyproject_toml, &filename, uv_version, writer)?; + // Ensure a deterministic order even when file walking changes + files.sort_unstable(); + Ok((filename, files)) +} + +fn write_wheel( + source_tree: &Path, + pyproject_toml: &PyProjectToml, + filename: &WheelFilename, + uv_version: &str, + mut wheel_writer: impl DirectoryWriter, +) -> Result<(), Error> { + let settings = pyproject_toml + .settings() + .cloned() + .unwrap_or_else(BuildBackendSettings::default); // Wheel excludes let mut excludes: Vec = Vec::new(); @@ -69,7 +115,7 @@ pub fn build_wheel( debug!("Wheel excludes: {:?}", excludes); let exclude_matcher = build_exclude_matcher(excludes)?; - debug!("Adding content files to {}", wheel_path.user_display()); + debug!("Adding content files to wheel"); if settings.module_root.is_absolute() { return Err(Error::AbsoluteModuleRoot(settings.module_root.clone())); } @@ -165,17 +211,17 @@ pub fn build_wheel( )?; } - debug!("Adding metadata files to: `{}`", wheel_path.user_display()); + debug!("Adding metadata files to wheel"); let dist_info_dir = write_dist_info( &mut wheel_writer, - &pyproject_toml, - &filename, + pyproject_toml, + filename, source_tree, uv_version, )?; wheel_writer.close(&dist_info_dir)?; - Ok(filename) + Ok(()) } /// Build a wheel from the source tree and place it in the output directory. @@ -384,7 +430,7 @@ fn wheel_subdir_from_globs( src: &Path, target: &str, globs: &[String], - wheel_writer: &mut ZipDirectoryWriter, + wheel_writer: &mut impl DirectoryWriter, // For error messages globs_field: &str, ) -> Result<(), Error> {