Nested namespace packages support (#10541)

## Summary
PEP 420 says [nested namespace
packages](https://peps.python.org/pep-0420/#nested-namespace-packages)
are allowed, i.e. marking a directory as a namespace package marks all
subdirectories in the subtree as namespace packages.

`is_package` is modified to use `Path::starts_with` and the order of
checks is reversed to do in-memory checks first before hitting the disk.

## Test Plan
Added unit tests. Previously all tests were run with `namespace_packages
== &[]`. Verified that one of the tests was failing before changing the
implementation.

## Future Improvements
The `is_package_with_cache` can probably be rewritten to avoid repeated
calls to `Path::starts_with`, by caching all directories up to the
`namespace_root`:
```ruff
let namespace_root = namespace_packages
    .iter()
    .filter(|namespace_package| path.starts_with(namespace_package))
    .min();
```

CONTRIBUTING.md

@@ -813,8 +813,8 @@ To understand Ruff's import categorization system, we first need to define two c
     "project root".)
 - "Package root": The top-most directory defining the Python package that includes a given Python
     file. To find the package root for a given Python file, traverse up its parent directories until
-    you reach a parent directory that doesn't contain an `__init__.py` file (and isn't marked as
+    you reach a parent directory that doesn't contain an `__init__.py` file (and isn't in a subtree
-    a [namespace package](https://docs.astral.sh/ruff/settings/#namespace-packages)); take the directory
+    marked as a [namespace package](https://docs.astral.sh/ruff/settings/#namespace-packages)); take the directory
     just before that, i.e., the first directory in the package.
 
 For example, given:

crates/ruff_linter/src/packaging.rs

@@ -26,17 +26,14 @@ use std::path::{Path, PathBuf};
 /// Return `true` if the directory at the given `Path` appears to be a Python
 /// package.
 pub fn is_package(path: &Path, namespace_packages: &[PathBuf]) -> bool {
-    path.join("__init__.py").is_file()
+    namespace_packages
-        || namespace_packages
+        .iter()
-            .iter()
+        .any(|namespace_package| path.starts_with(namespace_package))
-            .any(|namespace_package| namespace_package == path)
+        || path.join("__init__.py").is_file()
 }
 
-/// Return the package root for the given Python file.
+/// Return the package root for the given path to a directory with Python file.
-pub fn detect_package_root<'a>(
+pub fn detect_package_root<'a>(path: &'a Path, namespace_packages: &[PathBuf]) -> Option<&'a Path> {
-    path: &'a Path,
-    namespace_packages: &'a [PathBuf],
-) -> Option<&'a Path> {
     let mut current = None;
     for parent in path.ancestors() {
         if !is_package(parent, namespace_packages) {
@@ -84,4 +81,39 @@ mod tests {
             None,
         );
     }
+
+    #[test]
+    fn package_detection_with_namespace_packages() {
+        assert_eq!(
+            detect_package_root(&test_resource_path("project/python_modules/core/core"), &[],),
+            Some(test_resource_path("project/python_modules/core/core").as_path())
+        );
+
+        assert_eq!(
+            detect_package_root(
+                &test_resource_path("project/python_modules/core/core"),
+                &[test_resource_path("project/python_modules/core"),],
+            ),
+            Some(test_resource_path("project/python_modules/core").as_path())
+        );
+
+        assert_eq!(
+            detect_package_root(
+                &test_resource_path("project/python_modules/core/core"),
+                &[
+                    test_resource_path("project/python_modules/core"),
+                    test_resource_path("project/python_modules"),
+                ],
+            ),
+            Some(test_resource_path("project/python_modules").as_path())
+        );
+
+        assert_eq!(
+            detect_package_root(
+                &test_resource_path("project/python_modules/core/core"),
+                &[test_resource_path("project/python_modules"),],
+            ),
+            Some(test_resource_path("project/python_modules").as_path())
+        );
+    }
 }

crates/ruff_workspace/src/options.rs

@@ -293,8 +293,8 @@ pub struct Options {
     pub builtins: Option<Vec<String>>,
 
     /// Mark the specified directories as namespace packages. For the purpose of
-    /// module resolution, Ruff will treat those directories as if they
+    /// module resolution, Ruff will treat those directories and all their subdirectories
-    /// contained an `__init__.py` file.
+    /// as if they contained an `__init__.py` file.
     #[option(
         default = r#"[]"#,
         value_type = "list[str]",

crates/ruff_workspace/src/resolver.rs

@@ -203,7 +203,7 @@ impl<'a> Resolver<'a> {
 /// A wrapper around `detect_package_root` to cache filesystem lookups.
 fn detect_package_root_with_cache<'a>(
     path: &'a Path,
-    namespace_packages: &'a [PathBuf],
+    namespace_packages: &[PathBuf],
     package_cache: &mut FxHashMap<&'a Path, bool>,
 ) -> Option<&'a Path> {
     let mut current = None;
@@ -219,7 +219,7 @@ fn detect_package_root_with_cache<'a>(
 /// A wrapper around `is_package` to cache filesystem lookups.
 fn is_package_with_cache<'a>(
     path: &'a Path,
-    namespace_packages: &'a [PathBuf],
+    namespace_packages: &[PathBuf],
     package_cache: &mut FxHashMap<&'a Path, bool>,
 ) -> bool {
     *package_cache

ruff.schema.json

@@ -509,7 +509,7 @@
       ]
     },
     "namespace-packages": {
-      "description": "Mark the specified directories as namespace packages. For the purpose of module resolution, Ruff will treat those directories as if they contained an `__init__.py` file.",
+      "description": "Mark the specified directories as namespace packages. For the purpose of module resolution, Ruff will treat those directories and all their subdirectories as if they contained an `__init__.py` file.",
       "type": [
         "array",
         "null"