From ba14f6967639a75e3c0781659f646bd95beafdab Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 18 Mar 2024 20:06:16 -0700 Subject: [PATCH] Search in both `purelib` and `platlib` for site-packages population (#2537) ## Summary In reality, there's no such thing as the `site-packages` directory for a given virtualenv. Rather, Python defines both `purelib` and `platlib`, where the former is for pure-Python packages and the latter is for packages that contain native code. These are almost always set to the same thing... but they don't _have_ to be, and in fact of Fedora they are not. This PR changes the `site_packages` method to return an iterator of directories. Closes https://github.com/astral-sh/uv/issues/2527. --- crates/uv-installer/src/site_packages.rs | 82 ++++++++++--------- .../uv-interpreter/src/python_environment.rs | 15 +++- crates/uv/src/commands/mod.rs | 19 +++-- crates/uv/tests/pip_sync.rs | 2 +- 4 files changed, 67 insertions(+), 51 deletions(-) diff --git a/crates/uv-installer/src/site_packages.rs b/crates/uv-installer/src/site_packages.rs index 478e8223d..62a26ef6f 100644 --- a/crates/uv-installer/src/site_packages.rs +++ b/crates/uv-installer/src/site_packages.rs @@ -42,47 +42,51 @@ impl<'a> SitePackages<'a> { let mut by_name = FxHashMap::default(); let mut by_url = FxHashMap::default(); - // Read the site-packages directory. - let site_packages = match fs::read_dir(venv.site_packages()) { - Ok(site_packages) => site_packages, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - return Ok(Self { - venv, - distributions, - by_name, - by_url, - }); - } - Err(err) => return Err(err).context("Failed to read site-packages directory"), - }; - - // Index all installed packages by name. - for entry in site_packages { - let entry = entry?; - if entry.file_type()?.is_dir() { - let path = entry.path(); - - let Some(dist_info) = InstalledDist::try_from_path(&path) - .with_context(|| format!("Failed to read metadata: from {}", path.display()))? - else { - continue; - }; - - let idx = distributions.len(); - - // Index the distribution by name. - by_name - .entry(dist_info.name().clone()) - .or_insert_with(Vec::new) - .push(idx); - - // Index the distribution by URL. - if let Some(url) = dist_info.as_editable() { - by_url.entry(url.clone()).or_insert_with(Vec::new).push(idx); + for site_packages in venv.site_packages() { + // Read the site-packages directory. + let site_packages = match fs::read_dir(site_packages) { + Ok(site_packages) => site_packages, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Ok(Self { + venv, + distributions, + by_name, + by_url, + }); } + Err(err) => return Err(err).context("Failed to read site-packages directory"), + }; - // Add the distribution to the database. - distributions.push(Some(dist_info)); + // Index all installed packages by name. + for entry in site_packages { + let entry = entry?; + if entry.file_type()?.is_dir() { + let path = entry.path(); + + let Some(dist_info) = + InstalledDist::try_from_path(&path).with_context(|| { + format!("Failed to read metadata: from {}", path.display()) + })? + else { + continue; + }; + + let idx = distributions.len(); + + // Index the distribution by name. + by_name + .entry(dist_info.name().clone()) + .or_insert_with(Vec::new) + .push(idx); + + // Index the distribution by URL. + if let Some(url) = dist_info.as_editable() { + by_url.entry(url.clone()).or_insert_with(Vec::new).push(idx); + } + + // Add the distribution to the database. + distributions.push(Some(dist_info)); + } } } diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index b0415931b..275738378 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -88,9 +88,18 @@ impl PythonEnvironment { self.interpreter.sys_executable() } - /// Returns the path to the `site-packages` directory inside a virtual environment. - pub fn site_packages(&self) -> &Path { - self.interpreter.purelib() + /// Returns an iterator over the `site-packages` directories inside a virtual environment. + /// + /// In most cases, `purelib` and `platlib` will be the same, and so the iterator will contain + /// a single element; however, in some distributions, they may be different. + pub fn site_packages(&self) -> impl Iterator { + std::iter::once(self.interpreter.purelib()).chain( + if self.interpreter.purelib() == self.interpreter.platlib() { + None + } else { + Some(self.interpreter.platlib()) + }, + ) } /// Returns the path to the `bin` directory inside a virtual environment. diff --git a/crates/uv/src/commands/mod.rs b/crates/uv/src/commands/mod.rs index 52777b49a..2ff170a8f 100644 --- a/crates/uv/src/commands/mod.rs +++ b/crates/uv/src/commands/mod.rs @@ -129,14 +129,17 @@ pub(super) async fn compile_bytecode( printer: Printer, ) -> anyhow::Result<()> { let start = std::time::Instant::now(); - let files = compile_tree(venv.site_packages(), venv.python_executable(), cache.root()) - .await - .with_context(|| { - format!( - "Failed to bytecode compile {}", - venv.site_packages().simplified_display() - ) - })?; + let mut files = 0; + for site_packages in venv.site_packages() { + files += compile_tree(site_packages, venv.python_executable(), cache.root()) + .await + .with_context(|| { + format!( + "Failed to bytecode-compile Python file in: {}", + site_packages.simplified_display() + ) + })?; + } let s = if files == 1 { "" } else { "s" }; writeln!( printer.stderr(), diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index b5de8f21c..26e9e13c5 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -2948,7 +2948,7 @@ fn compile_invalid_pyc_invalidation_mode() -> Result<()> { Resolved 1 package in [TIME] Downloaded 1 package in [TIME] Installed 1 package in [TIME] - error: Failed to bytecode compile [SITE-PACKAGES] + error: Failed to bytecode-compile Python file in: [SITE-PACKAGES] Caused by: Python process stderr: Invalid value for PYC_INVALIDATION_MODE "bogus", valid are "TIMESTAMP", "CHECKED_HASH", "UNCHECKED_HASH": Caused by: Bytecode compilation failed, expected "[SITE-PACKAGES]/[FIRST-FILE]", received: ""