mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-07 13:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Multiple modules in namespace packages

Browse source 
Support multiple root modules in namespace packages by enumerating them:

```toml
[tool.uv.build-backend]
module-name = ["foo", "bar"]
```

This allows applications with multiple root packages without migrating to workspaces. Since those are regular module names (we iterate over them an process each one like a single module names), it allows combining dotted (namespace) names and regular names. It also technically allows combining regular and stub modules, even though this is even less recommends.

We don't recommend this structure (please use a workspace instead, or structure everything in one root module), but it reduces the number of cases that need `namespace = true`.

Fixes #14435
Fixes #14438
This commit is contained in:
konstin 2025-07-04 16:26:20 +02:00
parent bb738aeb44
commit 28a97b5588
9 changed files with 297 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -4785,6 +4785,7 @@ dependencies = [
"indoc",
"insta",
"itertools 0.14.0",
"rustc-hash",
"schemars",
"serde",
"sha2",

1
crates/uv-build-backend/Cargo.toml
View file

@ -31,6 +31,7 @@ flate2 = { workspace = true, default-features = false }
fs-err = { workspace = true }
globset = { workspace = true }
itertools = { workspace = true }
rustc-hash = { workspace = true }
schemars = { workspace = true, optional = true }
serde = { workspace = true }
sha2 = { workspace = true }

188
crates/uv-build-backend/src/lib.rs
View file

@ -22,6 +22,7 @@ use uv_normalize::PackageName;
use uv_pypi_types::{Identifier, IdentifierParseError};
use crate::metadata::ValidationError;
use crate::settings::ModuleName;
#[derive(Debug, Error)]
pub enum Error {
@ -184,7 +185,7 @@ fn check_metadata_directory(
Ok(())
}
/// Returns the source root and the module path with the `__init__.py[i]` below to it while
/// Returns the source root and the module path(s) with the `__init__.py[i]` below to it while
/// checking the project layout and names.
///
/// Some target platforms have case-sensitive filesystems, while others have case-insensitive
@ -198,13 +199,15 @@ fn check_metadata_directory(
/// dist-info-normalization, the rules are lowercasing, replacing `.` with `_` and
/// replace `-` with `_`. Since `.` and `-` are not allowed in identifiers, we can use a string
/// comparison with the module name.
///
/// While we recommend one module per package, it is possible to declare a list of modules.
fn find_roots(
source_tree: &Path,
pyproject_toml: &PyProjectToml,
relative_module_root: &Path,
module_name: Option<&str>,
module_name: Option<&ModuleName>,
namespace: bool,
) -> Result<(PathBuf, PathBuf), Error> {
) -> Result<(PathBuf, Vec<PathBuf>), Error> {
let relative_module_root = uv_fs::normalize_path(relative_module_root);
let src_root = source_tree.join(&relative_module_root);
if !src_root.starts_with(source_tree) {
@ -215,22 +218,45 @@ fn find_roots(
if namespace {
// `namespace = true` disables module structure checks.
let module_relative = if let Some(module_name) = module_name {
module_name.split('.').collect::<PathBuf>()
let modules_relative = if let Some(module_name) = module_name {
match module_name {
ModuleName::Name(name) => {
vec![name.split('.').collect::<PathBuf>()]
}
ModuleName::Names(names) => names
.iter()
.map(|name| name.split('.').collect::<PathBuf>())
.collect(),
}
} else {
PathBuf::from(pyproject_toml.name().as_dist_info_name().to_string())
vec![PathBuf::from(
pyproject_toml.name().as_dist_info_name().to_string(),
)]
};
debug!("Namespace module path: {}", module_relative.user_display());
return Ok((src_root, module_relative));
for module_relative in &modules_relative {
debug!("Namespace module path: {}", module_relative.user_display());
}
return Ok((src_root, modules_relative));
}
let module_relative = if let Some(module_name) = module_name {
module_path_from_module_name(&src_root, module_name)?
let modules_relative = if let Some(module_name) = module_name {
match module_name {
ModuleName::Name(name) => vec![module_path_from_module_name(&src_root, name)?],
ModuleName::Names(names) => names
.iter()
.map(|name| module_path_from_module_name(&src_root, name))
.collect::<Result<_, _>>()?,
}
} else {
find_module_path_from_package_name(&src_root, pyproject_toml.name())?
vec![find_module_path_from_package_name(
&src_root,
pyproject_toml.name(),
)?]
};
debug!("Module path: {}", module_relative.user_display());
Ok((src_root, module_relative))
for module_relative in &modules_relative {
debug!("Module path: {}", module_relative.user_display());
}
Ok((src_root, modules_relative))
}
/// Infer stubs packages from package name alone.
@ -410,6 +436,15 @@ mod tests {
})
}
fn build_err(source_root: &Path) -> String {
let dist = TempDir::new().unwrap();
let build_err = build(source_root, dist.path()).unwrap_err();
let err_message: String = format_err(&build_err)
.replace(&source_root.user_display().to_string(), "[TEMP_PATH]")
.replace('\\', "/");
err_message
}
fn sdist_contents(source_dist_path: &Path) -> Vec<String> {
let sdist_reader = BufReader::new(File::open(source_dist_path).unwrap());
let mut source_dist = tar::Archive::new(GzDecoder::new(sdist_reader));
@ -998,13 +1033,8 @@ mod tests {
fs_err::create_dir_all(src.path().join("src").join("simple_namespace").join("part"))
.unwrap();
let dist = TempDir::new().unwrap();
let build_err = build(src.path(), dist.path()).unwrap_err();
let err_message = format_err(&build_err)
.replace(&src.path().user_display().to_string(), "[TEMP_PATH]")
.replace('\\', "/");
assert_snapshot!(
err_message,
build_err(src.path()),
@"Expected a Python module at: `[TEMP_PATH]/src/simple_namespace/part/__init__.py`"
);
@ -1025,16 +1055,13 @@ mod tests {
.join("simple_namespace")
.join("__init__.py");
File::create(&bogus_init_py).unwrap();
let build_err = build(src.path(), dist.path()).unwrap_err();
let err_message = format_err(&build_err)
.replace(&src.path().user_display().to_string(), "[TEMP_PATH]")
.replace('\\', "/");
assert_snapshot!(
err_message,
build_err(src.path()),
@"For namespace packages, `__init__.py[i]` is not allowed in parent directory: `[TEMP_PATH]/src/simple_namespace`"
);
fs_err::remove_file(bogus_init_py).unwrap();
let dist = TempDir::new().unwrap();
let build1 = build(src.path(), dist.path()).unwrap();
assert_snapshot!(build1.source_dist_contents.join("\n"), @r"
simple_namespace_part-1.0.0/
@ -1209,4 +1236,117 @@ mod tests {
cloud_db_schema_stubs-1.0.0.dist-info/WHEEL
");
}
/// A package with multiple modules, one a regular module and two namespace modules.
#[test]
fn multiple_module_names() {
let src = TempDir::new().unwrap();
let pyproject_toml = indoc! {r#"
[project]
name = "simple-namespace-part"
version = "1.0.0"
[tool.uv.build-backend]
module-name = ["foo", "simple_namespace.part_a", "simple_namespace.part_b"]
[build-system]
requires = ["uv_build>=0.5.15,<0.6"]
build-backend = "uv_build"
"#
};
fs_err::write(src.path().join("pyproject.toml"), pyproject_toml).unwrap();
fs_err::create_dir_all(src.path().join("src").join("foo")).unwrap();
fs_err::create_dir_all(
src.path()
.join("src")
.join("simple_namespace")
.join("part_a"),
)
.unwrap();
fs_err::create_dir_all(
src.path()
.join("src")
.join("simple_namespace")
.join("part_b"),
)
.unwrap();
// Most of these checks exist in other tests too, but we want to ensure that they apply
// with multiple modules too.
// The first module is missing an `__init__.py`.
assert_snapshot!(
build_err(src.path()),
@"Expected a Python module at: `[TEMP_PATH]/src/foo/__init__.py`"
);
// Create the first correct `__init__.py` file
File::create(src.path().join("src").join("foo").join("__init__.py")).unwrap();
// The second module, a namespace, is missing an `__init__.py`.
assert_snapshot!(
build_err(src.path()),
@"Expected a Python module at: `[TEMP_PATH]/src/simple_namespace/part_a/__init__.py`"
);
// Create the other two correct `__init__.py` files
File::create(
src.path()
.join("src")
.join("simple_namespace")
.join("part_a")
.join("__init__.py"),
)
.unwrap();
File::create(
src.path()
.join("src")
.join("simple_namespace")
.join("part_b")
.join("__init__.py"),
)
.unwrap();
// For the second module, a namespace, there must not be an `__init__.py` here.
let bogus_init_py = src
.path()
.join("src")
.join("simple_namespace")
.join("__init__.py");
File::create(&bogus_init_py).unwrap();
assert_snapshot!(
build_err(src.path()),
@"For namespace packages, `__init__.py[i]` is not allowed in parent directory: `[TEMP_PATH]/src/simple_namespace`"
);
fs_err::remove_file(bogus_init_py).unwrap();
let dist = TempDir::new().unwrap();
let build = build(src.path(), dist.path()).unwrap();
assert_snapshot!(build.source_dist_contents.join("\n"), @r"
simple_namespace_part-1.0.0/
simple_namespace_part-1.0.0/PKG-INFO
simple_namespace_part-1.0.0/pyproject.toml
simple_namespace_part-1.0.0/src
simple_namespace_part-1.0.0/src/foo
simple_namespace_part-1.0.0/src/foo/__init__.py
simple_namespace_part-1.0.0/src/simple_namespace
simple_namespace_part-1.0.0/src/simple_namespace/part_a
simple_namespace_part-1.0.0/src/simple_namespace/part_a/__init__.py
simple_namespace_part-1.0.0/src/simple_namespace/part_b
simple_namespace_part-1.0.0/src/simple_namespace/part_b/__init__.py
");
assert_snapshot!(build.wheel_contents.join("\n"), @r"
foo/
foo/__init__.py
simple_namespace/
simple_namespace/part_a/
simple_namespace/part_a/__init__.py
simple_namespace/part_b/
simple_namespace/part_b/__init__.py
simple_namespace_part-1.0.0.dist-info/
simple_namespace_part-1.0.0.dist-info/METADATA
simple_namespace_part-1.0.0.dist-info/RECORD
simple_namespace_part-1.0.0.dist-info/WHEEL
");
}
}

19
crates/uv-build-backend/src/settings.rs
View file

@ -34,15 +34,19 @@ pub struct BuildBackendSettings {
/// For namespace packages with a single module, the path can be dotted, e.g., `foo.bar` or
/// `foo-stubs.bar`.
///
/// For namespace packages with multiple modules, the path can be a list, e.g.,
/// `["foo", "bar"]`. We recommend using a single module per package, splitting multiple
/// packages into a workspace.
///
/// Note that using this option runs the risk of creating two packages with different names but
/// the same module names. Installing such packages together leads to unspecified behavior,
/// often with corrupted files or directory trees.
#[option(
default = r#"None"#,
value_type = "str",
value_type = "str | list[str]",
example = r#"module-name = "sklearn""#
)]
pub module_name: Option<String>,
pub module_name: Option<ModuleName>,
/// Glob expressions which files and directories to additionally include in the source
/// distribution.
@ -181,6 +185,17 @@ impl Default for BuildBackendSettings {
}
}
/// Whether to include a single module or multiple modules.
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum ModuleName {
/// A single module name.
Name(String),
/// Multiple module names, which are all included.
Names(Vec<String>),
}
/// Data includes for wheels.
///
/// See `BuildBackendSettings::data`.

24
crates/uv-build-backend/src/source_dist.rs
View file

@ -68,22 +68,24 @@ fn source_dist_matcher(
includes.push(globset::escape("pyproject.toml"));
// Check that the source tree contains a module.
let (src_root, module_relative) = find_roots(
let (src_root, modules_relative) = find_roots(
source_tree,
pyproject_toml,
&settings.module_root,
settings.module_name.as_deref(),
settings.module_name.as_ref(),
settings.namespace,
)?;
// The wheel must not include any files included by the source distribution (at least until we
// have files generated in the source dist -> wheel build step).
let import_path = uv_fs::normalize_path(
&uv_fs::relative_to(src_root.join(module_relative), source_tree)
.expect("module root is inside source tree"),
)
.portable_display()
.to_string();
includes.push(format!("{}/**", globset::escape(&import_path)));
for module_relative in modules_relative {
// The wheel must not include any files included by the source distribution (at least until we
// have files generated in the source dist -> wheel build step).
let import_path = uv_fs::normalize_path(
&uv_fs::relative_to(src_root.join(module_relative), source_tree)
.expect("module root is inside source tree"),
)
.portable_display()
.to_string();
includes.push(format!("{}/**", globset::escape(&import_path)));
}
for include in includes {
let glob = PortableGlobParser::Uv
.parse(&include)

91
crates/uv-build-backend/src/wheel.rs
View file

@ -1,6 +1,7 @@
use fs_err::File;
use globset::{GlobSet, GlobSetBuilder};
use itertools::Itertools;
use rustc_hash::FxHashSet;
use sha2::{Digest, Sha256};
use std::io::{BufReader, Read, Write};
use std::path::{Path, PathBuf};
@ -127,55 +128,61 @@ fn write_wheel(
source_tree,
pyproject_toml,
&settings.module_root,
settings.module_name.as_deref(),
settings.module_name.as_ref(),
settings.namespace,
)?;
// For convenience, have directories for the whole tree in the wheel
for ancestor in module_relative.ancestors().skip(1) {
if ancestor == Path::new("") {
continue;
}
wheel_writer.write_directory(&ancestor.portable_display().to_string())?;
}
let mut files_visited = 0;
for entry in WalkDir::new(src_root.join(module_relative))
.sort_by_file_name()
.into_iter()
.filter_entry(|entry| !exclude_matcher.is_match(entry.path()))
{
let entry = entry.map_err(|err| Error::WalkDir {
root: source_tree.to_path_buf(),
err,
})?;
let mut prefix_directories = FxHashSet::default();
for module_relative in module_relative {
// For convenience, have directories for the whole tree in the wheel
for ancestor in module_relative.ancestors().skip(1) {
if ancestor == Path::new("") {
continue;
}
// Avoid duplicate directories in the zip.
if prefix_directories.insert(ancestor.to_path_buf()) {
wheel_writer.write_directory(&ancestor.portable_display().to_string())?;
}
}
files_visited += 1;
if files_visited > 10000 {
warn_user_once!(
"Visited more than 10,000 files for wheel build. \
for entry in WalkDir::new(src_root.join(module_relative))
.sort_by_file_name()
.into_iter()
.filter_entry(|entry| !exclude_matcher.is_match(entry.path()))
{
let entry = entry.map_err(|err| Error::WalkDir {
root: source_tree.to_path_buf(),
err,
})?;
files_visited += 1;
if files_visited > 10000 {
warn_user_once!(
"Visited more than 10,000 files for wheel build. \
Consider using more constrained includes or more excludes."
);
}
);
}
// We only want to take the module root, but since excludes start at the source tree root,
// we strip higher than we iterate.
let match_path = entry
.path()
.strip_prefix(source_tree)
.expect("walkdir starts with root");
let entry_path = entry
.path()
.strip_prefix(&src_root)
.expect("walkdir starts with root");
if exclude_matcher.is_match(match_path) {
trace!("Excluding from module: `{}`", match_path.user_display());
continue;
}
// We only want to take the module root, but since excludes start at the source tree root,
// we strip higher than we iterate.
let match_path = entry
.path()
.strip_prefix(source_tree)
.expect("walkdir starts with root");
let entry_path = entry
.path()
.strip_prefix(&src_root)
.expect("walkdir starts with root");
if exclude_matcher.is_match(match_path) {
trace!("Excluding from module: `{}`", match_path.user_display());
continue;
}
let entry_path = entry_path.portable_display().to_string();
debug!("Adding to wheel: {entry_path}");
wheel_writer.write_dir_entry(&entry, &entry_path)?;
let entry_path = entry_path.portable_display().to_string();
debug!("Adding to wheel: {entry_path}");
wheel_writer.write_dir_entry(&entry, &entry_path)?;
}
}
debug!("Visited {files_visited} files for wheel build");
@ -269,7 +276,7 @@ pub fn build_editable(
source_tree,
&pyproject_toml,
&settings.module_root,
settings.module_name.as_deref(),
settings.module_name.as_ref(),
settings.namespace,
)?;

25
docs/concepts/build-backend.md
View file

@ -134,16 +134,37 @@ the project structure:
pyproject.toml
src
├── foo
│   └── __init__.py
│ └── __init__.py
└── bar
└── __init__.py
```
While we do not recommend this structure (i.e., you should use a workspace with multiple packages
instead), it is supported via the `namespace` option:
instead), it is supported by passing a list to the `module-name` option:
```toml title="pyproject.toml"
[tool.uv.build-backend]
module-name = ["foo", "bar"]
```
The `namespace = true` option offers an opt-out to enumerating all modules for complex namespace
packages. Declaring an explicit `module-name` should be preferred over `namespace = true`.
```text
pyproject.toml
src
└── foo
├── bar
│ └── __init__.py
└── baz
└── __init__.py
```
And the configuration would be:
```toml title="pyproject.toml"
[tool.uv.build-backend]
module-name = "foo"
namespace = true
```

6
docs/reference/settings.md
View file

@ -474,13 +474,17 @@ being the module name, and which contain a `__init__.pyi` file.
For namespace packages with a single module, the path can be dotted, e.g., `foo.bar` or
`foo-stubs.bar`.
For namespace packages with multiple modules, the path can be a list, e.g.,
`["foo", "bar"]`. We recommend using a single module per package, splitting multiple
packages into a workspace.
Note that using this option runs the risk of creating two packages with different names but
the same module names. Installing such packages together leads to unspecified behavior,
often with corrupted files or directory trees.
**Default value**: `None`
**Type**: `str`
**Type**: `str | list[str]`
**Example usage**:

28
uv.schema.json generated
View file

@ -668,10 +668,14 @@
"default": true
},
"module-name": {
"description": "The name of the module directory inside `module-root`.\n\nThe default module name is the package name with dots and dashes replaced by underscores.\n\nPackage names need to be valid Python identifiers, and the directory needs to contain a\n`__init__.py`. An exception are stubs packages, whose name ends with `-stubs`, with the stem\nbeing the module name, and which contain a `__init__.pyi` file.\n\nFor namespace packages with a single module, the path can be dotted, e.g., `foo.bar` or\n`foo-stubs.bar`.\n\nNote that using this option runs the risk of creating two packages with different names but\nthe same module names. Installing such packages together leads to unspecified behavior,\noften with corrupted files or directory trees.",
"type": [
"string",
"null"
"description": "The name of the module directory inside `module-root`.\n\nThe default module name is the package name with dots and dashes replaced by underscores.\n\nPackage names need to be valid Python identifiers, and the directory needs to contain a\n`__init__.py`. An exception are stubs packages, whose name ends with `-stubs`, with the stem\nbeing the module name, and which contain a `__init__.pyi` file.\n\nFor namespace packages with a single module, the path can be dotted, e.g., `foo.bar` or\n`foo-stubs.bar`.\n\nFor namespace packages with multiple modules, the path can be a list, e.g.,\n`[\"foo\", \"bar\"]`. We recommend using a single module per package, splitting multiple\npackages into a workspace.\n\nNote that using this option runs the risk of creating two packages with different names but\nthe same module names. Installing such packages together leads to unspecified behavior,\noften with corrupted files or directory trees.",
"anyOf": [
{
"$ref": "#/definitions/ModuleName"
},
{
"type": "null"
}
],
"default": null
},
@ -1052,6 +1056,22 @@
"description": "A PEP 508-compliant marker expression, e.g., `sys_platform == 'Darwin'`",
"type": "string"
},
"ModuleName": {
"description": "Whether to include a single module or multiple modules.",
"anyOf": [
{
"description": "A single module name.",
"type": "string"
},
{
"description": "Multiple module names, which are all included.",
"type": "array",
"items": {
"type": "string"
}
}
]
},
"PackageName": {
"description": "The normalized name of a package.\n\nConverts the name to lowercase and collapses runs of `-`, `_`, and `.` down to a single `-`.\nFor example, `---`, `.`, and `__` are all converted to a single `-`.\n\nSee: <https://packaging.python.org/en/latest/specifications/name-normalization/>",
"type": "string"