Build backend: Warn when visiting over 10k files (#9523)
Some checks are pending
CI / build binary | linux (push) Blocked by required conditions
CI / build binary | macos aarch64 (push) Blocked by required conditions
CI / build binary | macos x86_64 (push) Blocked by required conditions
CI / check system | python3.13 on windows (push) Blocked by required conditions
CI / check system | python3.12 via chocolatey (push) Blocked by required conditions
CI / check system | conda3.11 on windows (push) Blocked by required conditions
CI / integration test | conda on ubuntu (push) Blocked by required conditions
CI / integration test | free-threaded on linux (push) Blocked by required conditions
CI / integration test | free-threaded on windows (push) Blocked by required conditions
CI / integration test | pypy on ubuntu (push) Blocked by required conditions
CI / integration test | pypy on windows (push) Blocked by required conditions
CI / integration test | graalpy on ubuntu (push) Blocked by required conditions
CI / integration test | graalpy on windows (push) Blocked by required conditions
CI / integration test | github actions (push) Blocked by required conditions
CI / integration test | determine publish changes (push) Blocked by required conditions
CI / integration test | uv publish (push) Blocked by required conditions
CI / check cache | ubuntu (push) Blocked by required conditions
CI / check cache | macos aarch64 (push) Blocked by required conditions
CI / check system | python on debian (push) Blocked by required conditions
CI / check system | python on fedora (push) Blocked by required conditions
CI / check system | python on ubuntu (push) Blocked by required conditions
CI / check system | python on opensuse (push) Blocked by required conditions
CI / check system | amazonlinux (push) Blocked by required conditions
CI / check system | python on rocky linux 8 (push) Blocked by required conditions
CI / Determine changes (push) Waiting to run
CI / lint (push) Waiting to run
CI / cargo clippy | ubuntu (push) Blocked by required conditions
CI / cargo clippy | windows (push) Blocked by required conditions
CI / cargo dev generate-all (push) Blocked by required conditions
CI / cargo shear (push) Waiting to run
CI / check system | python on rocky linux 9 (push) Blocked by required conditions
CI / check system | pypy on ubuntu (push) Blocked by required conditions
CI / check system | pyston (push) Blocked by required conditions
CI / cargo test | ubuntu (push) Blocked by required conditions
CI / cargo test | macos (push) Blocked by required conditions
CI / check system | alpine (push) Blocked by required conditions
CI / check system | python on macos aarch64 (push) Blocked by required conditions
CI / check system | homebrew python on macos aarch64 (push) Blocked by required conditions
CI / check system | python on macos x86_64 (push) Blocked by required conditions
CI / check system | python3.10 on windows (push) Blocked by required conditions
CI / check system | python3.10 on windows x86 (push) Blocked by required conditions
CI / cargo test | windows (push) Blocked by required conditions
CI / check windows trampoline | aarch64 (push) Blocked by required conditions
CI / check windows trampoline | i686 (push) Blocked by required conditions
CI / check windows trampoline | x86_64 (push) Blocked by required conditions
CI / test windows trampoline | i686 (push) Blocked by required conditions
CI / test windows trampoline | x86_64 (push) Blocked by required conditions
CI / typos (push) Waiting to run
CI / mkdocs (push) Waiting to run
CI / build binary | windows (push) Blocked by required conditions
CI / cargo build (msrv) (push) Blocked by required conditions
CI / check system | conda3.8 on windows (push) Blocked by required conditions
CI / build binary | freebsd (push) Blocked by required conditions
CI / ecosystem test | prefecthq/prefect (push) Blocked by required conditions
CI / ecosystem test | pallets/flask (push) Blocked by required conditions
CI / check system | python3.9 via pyenv (push) Blocked by required conditions
CI / check system | python3.13 (push) Blocked by required conditions
CI / check system | conda3.11 on linux (push) Blocked by required conditions
CI / check system | conda3.8 on linux (push) Blocked by required conditions
CI / check system | conda3.11 on macos (push) Blocked by required conditions
CI / check system | conda3.8 on macos (push) Blocked by required conditions
CI / check system | embedded python3.10 on windows (push) Blocked by required conditions
CI / benchmarks (push) Blocked by required conditions

Also log how many files we visited in total as a debugging aid.

No test for this one since we don't want to create 10k on disk in our
test suite.
This commit is contained in:
konsti 2024-11-29 16:46:11 +01:00 committed by GitHub
parent 590294a0ac
commit b9b37a9bab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -17,6 +17,7 @@ use tracing::{debug, trace};
use uv_distribution_filename::{SourceDistExtension, SourceDistFilename, WheelFilename};
use uv_fs::Simplified;
use uv_globfilter::{parse_portable_glob, GlobDirFilter, PortableGlobError};
use uv_warnings::warn_user_once;
use walkdir::{DirEntry, WalkDir};
use zip::{CompressionMethod, ZipWriter};
@ -345,6 +346,7 @@ pub fn build_wheel(
if !module_root.join("__init__.py").is_file() {
return Err(Error::MissingModule(module_root));
}
let mut files_visited = 0;
for entry in WalkDir::new(module_root)
.into_iter()
.filter_entry(|entry| !exclude_matcher.is_match(entry.path()))
@ -354,6 +356,14 @@ pub fn build_wheel(
err,
})?;
files_visited += 1;
if files_visited > 10000 {
warn_user_once!(
"Visited more than 10,000 files for wheel build. \
Consider using more constrained includes or more excludes."
);
}
let relative_path = entry
.path()
.strip_prefix(&strip_root)
@ -377,6 +387,7 @@ pub fn build_wheel(
));
}
}
debug!("Visited {files_visited} files for wheel build");
// Add the license files
if let Some(license_files) = &pyproject_toml.license_files() {
@ -712,8 +723,7 @@ pub fn build_source_dist(
let exclude_matcher = build_exclude_matcher(&settings.exclude)?;
// TODO(konsti): Add files linked by pyproject.toml
let mut files_visited = 0;
for entry in WalkDir::new(source_tree).into_iter().filter_entry(|entry| {
// TODO(konsti): This should be prettier.
let relative = entry
@ -733,6 +743,14 @@ pub fn build_source_dist(
root: source_tree.to_path_buf(),
err,
})?;
files_visited += 1;
if files_visited > 10000 {
warn_user_once!(
"Visited more than 10,000 files for source distribution build. \
Consider using more constrained includes or more excludes."
);
}
// TODO(konsti): This should be prettier.
let relative = entry
.path()
@ -746,6 +764,7 @@ pub fn build_source_dist(
add_source_dist_entry(&mut tar, &entry, &top_level, &source_dist_path, relative)?;
}
debug!("Visited {files_visited} files for source dist build");
tar.finish()
.map_err(|err| Error::TarWrite(source_dist_path.clone(), err))?;