mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 19:08:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Improve build backend excludes (#9281)

Browse source 
This PR contains three smaller improvements:
* Improve the include/exclude logging. We're still showing the current
directory as empty backticks, not sure what to do about that
* Add early stopping to license file globbing, so we don't traverse the
whole directory recursively when license files can only be in few places
* Support explicit wheel excludes. These are still not entirely right,
but at least we're correctly excluding compiled python files now. The
next step is to make sure that the wheel excludes contain all pattern
from source dist excludes, to make sure source tree -> wheel can't have
more files than source tree -> source dist -> wheel.
This commit is contained in:
konsti 2024-11-21 12:20:29 +01:00 • committed by GitHub
parent 3143494ddb
commit e49ecd868d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 107 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

112
crates/uv-build-backend/src/lib.rs
View file

@ -4,7 +4,7 @@ use crate::metadata::{PyProjectToml, ValidationError};
use flate2::write::GzEncoder;
use flate2::Compression;
use fs_err::File;
use globset::{Glob, GlobSetBuilder};
use globset::{Glob, GlobSet, GlobSetBuilder};
use itertools::Itertools;
use sha2::{Digest, Sha256};
use std::fs::FileType;
@ -318,6 +318,20 @@ pub fn build_wheel(
debug!("Writing wheel at {}", wheel_path.user_display());
let mut wheel_writer = ZipDirectoryWriter::new_wheel(File::create(&wheel_path)?);
// Wheel excludes
// TODO(konstin): The must be stronger than the source dist excludes, otherwise we can get more
// files in source tree -> wheel than for source tree -> source dist -> wheel.
let default_excludes: &[String] = &[
"__pycache__".to_string(),
"*.pyc".to_string(),
"*.pyo".to_string(),
];
let excludes = pyproject_toml
.wheel_settings()
.and_then(|settings| settings.exclude.as_deref())
.unwrap_or(default_excludes);
let exclude_matcher = build_exclude_matcher(excludes)?;
debug!("Adding content files to {}", wheel_path.user_display());
let module_root = pyproject_toml
.wheel_settings()
@ -331,7 +345,10 @@ pub fn build_wheel(
if !module_root.join("__init__.py").is_file() {
return Err(Error::MissingModule(module_root));
}
for entry in WalkDir::new(module_root) {
for entry in WalkDir::new(module_root)
.into_iter()
.filter_entry(|entry| !exclude_matcher.is_match(entry.path()))
{
let entry = entry.map_err(|err| Error::WalkDir {
root: source_tree.to_path_buf(),
err,
@ -340,9 +357,11 @@ pub fn build_wheel(
let relative_path = entry
.path()
.strip_prefix(&strip_root)
.expect("walkdir starts with root")
.user_display()
.to_string();
.expect("walkdir starts with root");
if exclude_matcher.is_match(relative_path) {
trace!("Excluding from module: `{}`", relative_path.user_display());
}
let relative_path = relative_path.user_display().to_string();
debug!("Adding to wheel: `{relative_path}`");
@ -484,7 +503,12 @@ fn wheel_subdir_from_globs(
let license_files_globs: Vec<_> = globs
.iter()
.map(|license_files| {
trace!("Including license files at: `{license_files}`");
trace!(
"Including {} at `{}` with `{}`",
globs_field,
src.user_display(),
license_files
);
parse_portable_glob(license_files)
})
.collect::<Result<_, _>>()
@ -492,7 +516,7 @@ fn wheel_subdir_from_globs(
field: globs_field.to_string(),
source: err,
})?;
let license_files_matcher =
let matcher =
GlobDirFilter::from_globs(&license_files_globs).map_err(|err| Error::GlobSetTooLarge {
field: globs_field.to_string(),
source: err,
@ -508,7 +532,7 @@ fn wheel_subdir_from_globs(
.expect("walkdir starts with root");
// Fast path: Don't descend into a directory that can't be included.
license_files_matcher.match_directory(relative)
matcher.match_directory(relative)
}) {
let entry = entry.map_err(|err| Error::WalkDir {
root: src.to_path_buf(),
@ -520,8 +544,8 @@ fn wheel_subdir_from_globs(
.strip_prefix(src)
.expect("walkdir starts with root");
if !license_files_matcher.match_path(relative) {
trace!("Excluding {}", relative.user_display());
if !matcher.match_path(relative) {
trace!("Excluding {}: `{}`", globs_field, relative.user_display());
continue;
};
@ -686,26 +710,7 @@ pub fn build_source_dist(
source: err,
})?;
let mut exclude_builder = GlobSetBuilder::new();
for exclude in settings.exclude {
// Excludes are unanchored
let exclude = if let Some(exclude) = exclude.strip_prefix("/") {
exclude.to_string()
} else {
format!("**/{exclude}").to_string()
};
let glob = parse_portable_glob(&exclude).map_err(|err| Error::PortableGlob {
field: "tool.uv.source-dist.exclude".to_string(),
source: err,
})?;
exclude_builder.add(glob);
}
let exclude_matcher = exclude_builder
.build()
.map_err(|err| Error::GlobSetTooLarge {
field: "tool.uv.source-dist.exclude".to_string(),
source: err,
})?;
let exclude_matcher = build_exclude_matcher(&settings.exclude)?;
// TODO(konsti): Add files linked by pyproject.toml
@ -735,7 +740,7 @@ pub fn build_source_dist(
.expect("walkdir starts with root");
if !include_matcher.match_path(relative) || exclude_matcher.is_match(relative) {
trace!("Excluding {}", relative.user_display());
trace!("Excluding: `{}`", relative.user_display());
continue;
};
@ -748,6 +753,31 @@ pub fn build_source_dist(
Ok(filename)
}
/// Build a globset matcher for excludes.
fn build_exclude_matcher(excludes: &[String]) -> Result<GlobSet, Error> {
let mut exclude_builder = GlobSetBuilder::new();
for exclude in excludes {
// Excludes are unanchored
let exclude = if let Some(exclude) = exclude.strip_prefix("/") {
exclude.to_string()
} else {
format!("**/{exclude}").to_string()
};
let glob = parse_portable_glob(&exclude).map_err(|err| Error::PortableGlob {
field: "tool.uv.source-dist.exclude".to_string(),
source: err,
})?;
exclude_builder.add(glob);
}
let exclude_matcher = exclude_builder
.build()
.map_err(|err| Error::GlobSetTooLarge {
field: "tool.uv.source-dist.exclude".to_string(),
source: err,
})?;
Ok(exclude_matcher)
}
/// Add a file or a directory to a source distribution.
fn add_source_dist_entry(
tar: &mut tar::Builder<GzEncoder<File>>,
@ -1107,6 +1137,12 @@ mod tests {
fs_err::copy(built_by_uv.join(dir), src.path().join(dir)).unwrap();
}
// Add some files to be excluded
let module_root = src.path().join("src").join("built_by_uv");
fs_err::create_dir_all(module_root.join("__pycache__")).unwrap();
File::create(module_root.join("__pycache__").join("compiled.pyc")).unwrap();
File::create(module_root.join("arithmetic").join("circle.pyc")).unwrap();
// Build a wheel from the source tree
let direct_output_dir = TempDir::new().unwrap();
build_wheel(src.path(), direct_output_dir.path(), None, "1.0.0+test").unwrap();
@ -1159,13 +1195,6 @@ mod tests {
)
.unwrap();
// Check that we write deterministic wheels.
let wheel_filename = "built_by_uv-0.1.0-py3-none-any.whl";
assert_eq!(
fs_err::read(direct_output_dir.path().join(wheel_filename)).unwrap(),
fs_err::read(indirect_output_dir.path().join(wheel_filename)).unwrap()
);
// Check the contained files and directories
assert_snapshot!(source_dist_contents.iter().map(|path| path.replace('\\', "/")).join("\n"), @r"
built_by_uv-0.1.0/LICENSE-APACHE
@ -1220,5 +1249,12 @@ mod tests {
built_by_uv/arithmetic/circle.py
built_by_uv/arithmetic/pi.txt
");
// Check that we write deterministic wheels.
let wheel_filename = "built_by_uv-0.1.0-py3-none-any.whl";
assert_eq!(
fs_err::read(direct_output_dir.path().join(wheel_filename)).unwrap(),
fs_err::read(indirect_output_dir.path().join(wheel_filename)).unwrap()
);
}
}

43
crates/uv-build-backend/src/metadata.rs
View file

@ -1,5 +1,5 @@
use crate::Error;
use globset::{Glob, GlobSetBuilder};
use globset::Glob;
use itertools::Itertools;
use serde::Deserialize;
use std::collections::{BTreeMap, Bound};
@ -8,7 +8,7 @@ use std::path::{Path, PathBuf};
use std::str::FromStr;
use tracing::{debug, trace};
use uv_fs::Simplified;
use uv_globfilter::parse_portable_glob;
use uv_globfilter::{parse_portable_glob, GlobDirFilter};
use uv_normalize::{ExtraName, PackageName};
use uv_pep440::{Version, VersionSpecifiers};
use uv_pep508::{Requirement, VersionOrUrl};
@ -328,7 +328,7 @@ impl PyProjectToml {
};
let mut license_files = Vec::new();
let mut license_glob_builder = GlobSetBuilder::new();
let mut license_globs_parsed = Vec::new();
for license_glob in license_globs {
let pep639_glob =
parse_portable_glob(license_glob).map_err(|err| Error::PortableGlob {
@ -341,28 +341,38 @@ impl PyProjectToml {
.join(pep639_glob.to_string())
.to_string_lossy()
.to_string();
license_glob_builder.add(Glob::new(&absolute_glob).map_err(|err| {
license_globs_parsed.push(Glob::new(&absolute_glob).map_err(|err| {
Error::GlobSet {
field: "project.license-files".to_string(),
err,
}
})?);
}
let license_globs = license_glob_builder.build().map_err(|err| Error::GlobSet {
field: "project.license-files".to_string(),
err,
})?;
let license_globs =
GlobDirFilter::from_globs(&license_globs_parsed).map_err(|err| {
Error::GlobSetTooLarge {
field: "tool.uv.source-dist.include".to_string(),
source: err,
}
})?;
for entry in WalkDir::new(".") {
for entry in WalkDir::new(root).into_iter().filter_entry(|entry| {
license_globs.match_directory(
entry
.path()
.strip_prefix(root)
.expect("walkdir starts with root"),
)
}) {
let entry = entry.map_err(|err| Error::WalkDir {
root: PathBuf::from("."),
err,
})?;
let relative = entry
.path()
.strip_prefix("./")
.strip_prefix(root)
.expect("walkdir starts with root");
if !license_globs.is_match(relative) {
if !license_globs.match_path(relative) {
trace!("Not a license files match: `{}`", relative.user_display());
continue;
}
@ -711,6 +721,17 @@ pub(crate) struct WheelSettings {
/// The directory that contains the module directory, usually `src`, or an empty path when
/// using the flat layout over the src layout.
pub(crate) module_root: Option<PathBuf>,
/// Glob expressions which files and directories to exclude from the previous source
/// distribution includes.
///
/// Excludes are not anchored, which means that `__pycache__` excludes all directories named
/// `__pycache__` and it's children anywhere. To anchor a directory, use a `/` prefix, e.g.,
/// `/dist` will exclude only `<project root>/dist`.
///
/// The glob syntax is the reduced portable glob from
/// [PEP 639](https://peps.python.org/pep-0639/#add-license-FILES-key).
pub(crate) exclude: Option<Vec<String>>,
/// Data includes for wheels.
pub(crate) data: Option<WheelDataIncludes>,
}