Finish install-many with pypi 10k most dependents (#732)

This PR combines three small changes to finish up the install-many
testing.

* Download pypi_10k_most_dependents.txt in script I'd like to have the
setup process of the large scale checks automated.
* Some install-many dev script improvements 
* Fix mkl_fft-1.3.6-58-cp310-cp310-manylinux2014_x86_64.whl:
mkl_fft-1.3.6-58-cp310-cp310-manylinux2014_x86_64.whl has multiple
Wheel-Version entries, we have to ignore that like pip

Apart from the mkl-fft fix the only other errors i've seen showing up
are
https://github.com/astral-sh/puffin/issues/520#issuecomment-1869625642.

crates/distribution-types/src/cached.rs

@@ -110,6 +110,13 @@ impl CachedDist {
             CachedDist::Url(dist) => dist.editable,
         }
     }
+
+    pub fn filename(&self) -> &WheelFilename {
+        match self {
+            CachedDist::Registry(dist) => &dist.filename,
+            CachedDist::Url(dist) => &dist.filename,
+        }
+    }
 }
 
 impl CachedDirectUrlDist {

crates/install-wheel-rs/src/wheel.rs

@@ -394,17 +394,18 @@ pub(crate) fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
     // {distribution}-{version}.dist-info/WHEEL is metadata about the archive itself in the same basic key: value format:
     let data = parse_key_value_file(&mut wheel_text.as_bytes(), "WHEEL")?;
 
-    let Some(wheel_version) = data.get("Wheel-Version").and_then(|wheel_versions| {
+    // mkl_fft-1.3.6-58-cp310-cp310-manylinux2014_x86_64.whl has multiple Wheel-Version entries, we have to ignore that
-        if let [wheel_version] = wheel_versions.as_slice() {
+    // like pip
-            wheel_version.split_once('.')
+    let wheel_version = data
-        } else {
+        .get("Wheel-Version")
-            None
+        .and_then(|wheel_versions| wheel_versions.first());
-        }
+    let wheel_version = wheel_version
-    }) else {
+        .and_then(|wheel_version| wheel_version.split_once('.'))
-        return Err(Error::InvalidWheel(
+        .ok_or_else(|| {
-            "Invalid Wheel-Version in WHEEL file".to_string(),
+            Error::InvalidWheel(format!(
-        ));
+                "Invalid Wheel-Version in WHEEL file: {wheel_version:?}"
-    };
+            ))
+        })?;
     // pip has some test wheels that use that ancient version,
     // and technically we only need to check that the version is not higher
     if wheel_version == ("0", "1") {

crates/puffin-dev/src/install_many.rs

@@ -71,7 +71,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> {
 
     for (idx, requirements) in requirements.chunks(100).enumerate() {
         info!("Chunk {idx}");
-        install_chunk(
+        if let Err(err) = install_chunk(
             requirements,
             &build_dispatch,
             tags,
@@ -79,7 +79,13 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> {
             &venv,
             &index_urls,
         )
-        .await?;
+        .await
+        {
+            eprintln!("💥 Chunk {idx} failed");
+            for cause in err.chain() {
+                eprintln!("  Caused by: {cause}");
+            }
+        }
     }
 
     Ok(())
@@ -102,7 +108,9 @@ async fn install_chunk(
     for failure in &failures {
         info!("Failed to find wheel: {failure}");
     }
-    info!("Failed to find {} wheel(s)", failures.len());
+    if !failures.is_empty() {
+        info!("Failed to find {} wheel(s)", failures.len());
+    }
     let wheels_and_source_dist = resolution.len();
     let resolution = if build_dispatch.no_build() {
         let only_wheels: FxHashMap<_, _> = resolution
@@ -150,7 +158,9 @@ async fn install_chunk(
     for failure in &failures {
         info!("Failed to fetch wheel: {failure}");
     }
-    info!("Failed to fetch {} wheel(s)", failures.len());
+    if !failures.is_empty() {
+        info!("Failed to fetch {} wheel(s)", failures.len());
+    }
 
     let wheels: Vec<_> = wheels.into_iter().chain(cached).collect();
     puffin_installer::Installer::new(venv)

crates/puffin-installer/src/downloader.rs

@@ -152,7 +152,7 @@ impl<'a, Context: BuildContext + Send + Sync> Downloader<'a, Context> {
     }
 
     /// Download, build, and unzip a single wheel.
-    #[instrument(skip_all, fields(name = %dist, url = dist.file().unwrap().url))]
+    #[instrument(skip_all, fields(name = %dist, size = ?dist.size(), url = dist.file().unwrap().url))]
     pub async fn get_wheel(
         &self,
         dist: Dist,

crates/puffin-installer/src/installer.rs

@@ -55,7 +55,7 @@ impl<'a> Installer<'a> {
                         .as_ref(),
                     self.link_mode,
                 )
-                .with_context(|| format!("Failed to install: {wheel}"))?;
+                .with_context(|| format!("Failed to install: {} ({wheel})", wheel.filename()))?;
 
                 if let Some(reporter) = self.reporter.as_ref() {
                     reporter.on_install_progress(wheel);

scripts/popular_packages/download.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+curl https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json | jq -r ".rows | .[].project" > pypi_8k_downloads.txt
+curl https://gist.githubusercontent.com/charliermarsh/07afd9f543dfea68408a4a42cede4be4/raw/6639cd58a2e10d6bb7821f891f00322c8630b60a/pypi_10k_most_dependents.txt > pypi_10k_most_dependents.txt

scripts/popular_packages/pypi_8k_downloads.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-curl https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json | jq -r ".rows | .[].project" > pypi_8k_downloads.txt
-