Special case missing header build errors (on linux) (#354)

One of the most common errors i observed are build failures due to
missing header files. On ubuntu, this generally means that you need to
install some `<...>-dev` package that the documentation tells you about,
e.g. [mysqlclient](https://github.com/PyMySQL/mysqlclient#linux) needs
`default-libmysqlclient-dev`, [some psycopg
versions](https://www.psycopg.org/psycopg3/docs/basic/install.html#local-installation)
(i remember that this was always required at some earlier point) require
`libpq-dev` and pygraphviz wants `graphviz-dev`. This is quite common
for many scientific packages (where conda has an advantage because they
can provide those package as a dependency).

The error message can be completely inscrutable if you're just a python
programmer (or user) and not a c programmer (example: pygraphviz):

```
warning: no files found matching '*.png' under directory 'doc'
warning: no files found matching '*.txt' under directory 'doc'
warning: no files found matching '*.css' under directory 'doc'
warning: no previously-included files matching '*~' found anywhere in distribution
warning: no previously-included files matching '*.pyc' found anywhere in distribution
warning: no previously-included files matching '.svn' found anywhere in distribution
no previously-included directories found matching 'doc/build'
pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory
 3020 | #include "graphviz/cgraph.h"
      |          ^~~~~~~~~~~~~~~~~~~
compilation terminated.
error: command '/usr/bin/gcc' failed with exit code 1
```

The only relevant part is `Fatal error: graphviz/cgraph.h: No such file
or directory`. Why is this file not there and how do i get it to be
there?

This is even harder to spot in pip's output, where it's 11 lines above
the last line:


![image](7a3d7279-e7b1-4511-ab22-d0a35be5e672)

I've special cased missing headers and made sure that the last line
tells you the important information: We're missing some header, please
check the documentation of {package} {version} for what to install:


![image](4bbb8923-5a82-472f-ab1f-9e1471aa2896)

Scrolling up:


![image](89a2495a-e188-4288-b534-ad885ee08763)

The difference gets even clearer with a default ubuntu terminal with its
80 columns:


![image](49fb27bc-07c6-4b10-a1a1-30ec8e112438)

---

Note that the situation is better for a missing compiler, there i get:

```
[...]
warning: no previously-included files matching '*~' found anywhere in distribution
warning: no previously-included files matching '*.pyc' found anywhere in distribution
warning: no previously-included files matching '.svn' found anywhere in distribution
no previously-included directories found matching 'doc/build'
error: command 'gcc' failed: No such file or directory
---
```
Putting the last line into google, the first two results tell me to
`sudo apt-get install gcc`, the third even tells me about `sudo apt
install build-essential`

Cargo.lock

@@ -2176,11 +2176,14 @@ dependencies = [
  "fs-err",
  "gourgeist",
  "indoc",
+ "insta",
+ "once_cell",
  "pep508_rs",
  "platform-host",
  "puffin-interpreter",
  "puffin-traits",
  "pyproject-toml",
+ "regex",
  "serde",
  "serde_json",
  "tar",

crates/install-wheel-rs/src/linker.rs

@@ -61,7 +61,7 @@ pub fn install_wheel(
     let wheel_file_path = wheel
         .as_ref()
         .join(format!("{dist_info_prefix}.dist-info/WHEEL"));
-    let wheel_text = std::fs::read_to_string(&wheel_file_path)?;
+    let wheel_text = std::fs::read_to_string(wheel_file_path)?;
     parse_wheel_version(&wheel_text)?;
 
     // > 1.c If Root-Is-Purelib == ‘true’, unpack archive into purelib (site-packages).

crates/puffin-build/Cargo.toml

@@ -21,7 +21,9 @@ anyhow = { workspace = true }
 flate2 = { workspace = true }
 fs-err = { workspace = true }
 indoc = { workspace = true }
+once_cell = { workspace = true }
 pyproject-toml = { workspace = true }
+regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tar = { workspace = true }
@@ -32,3 +34,6 @@ toml = { workspace = true }
 tracing = { workspace = true }
 which = { workspace = true}
 zip = { workspace = true }
+
+[dev-dependencies]
+insta = { version = "1.34.0" }

crates/puffin-build/src/lib.rs

@@ -2,6 +2,7 @@
 //!
 //! <https://packaging.python.org/en/latest/specifications/source-distribution-format/>
 
+use std::fmt::{Display, Formatter};
 use std::io;
 use std::io::BufRead;
 use std::path::{Path, PathBuf};
@@ -13,7 +14,9 @@ use flate2::read::GzDecoder;
 use fs_err as fs;
 use fs_err::{DirEntry, File};
 use indoc::formatdoc;
+use once_cell::sync::Lazy;
 use pyproject_toml::{BuildSystem, Project};
+use regex::Regex;
 use serde::{Deserialize, Serialize};
 use tar::Archive;
 use tempfile::{tempdir, TempDir};
@@ -26,6 +29,14 @@ use pep508_rs::Requirement;
 use puffin_interpreter::{InterpreterInfo, Virtualenv};
 use puffin_traits::BuildContext;
 
+/// e.g. `pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory`
+static MISSING_HEADER_RE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(
+        r".*\.(c|c..|h|h..):\d+:\d+: fatal error: (?<header>.*\.(h|h..)): No such file or directory"
+    )
+    .unwrap()
+});
+
 #[derive(Error, Debug)]
 pub enum Error {
     #[error(transparent)]
@@ -50,14 +61,62 @@ pub enum Error {
         stdout: String,
         stderr: String,
     },
+    /// Nudge the user towards installing the missing dev library
+    #[error("{message}:\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---")]
+    MissingHeader {
+        message: String,
+        stdout: String,
+        stderr: String,
+        #[source]
+        missing_header_cause: MissingHeaderCause,
+    },
+}
+
+#[derive(Debug, Error)]
+pub struct MissingHeaderCause {
+    header: String,
+    // I've picked this over the better readable package name to make clear that you need to
+    // look for the build dependencies of that version or git commit respectively
+    package_id: String,
+}
+
+impl Display for MissingHeaderCause {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "This error likely indicates that you need to install a library that provides \"{}\" for {}",
+            self.header, self.package_id
+        )
+    }
 }
 
 impl Error {
-    fn from_command_output(message: String, output: &Output) -> Self {
+    fn from_command_output(message: String, output: &Output, package_id: &str) -> Self {
+        let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
+        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
+
+        // In the cases i've seen it was the 5th last line (see test case), 10 seems like a
+        // reasonable cutoff
+        if let Some(header) =
+            stderr.lines().rev().take(10).find_map(|line| {
+                Some(MISSING_HEADER_RE.captures(line.trim())?["header"].to_string())
+            })
+        {
+            return Self::MissingHeader {
+                message,
+                stdout,
+                stderr,
+                missing_header_cause: MissingHeaderCause {
+                    header,
+                    package_id: package_id.to_string(),
+                },
+            };
+        }
+
         Self::BuildBackend {
             message,
-            stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
-            stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
+            stdout,
+            stderr,
         }
     }
 }
@@ -124,17 +183,22 @@ pub struct SourceBuild {
     /// > directory created by prepare_metadata_for_build_wheel, including any unrecognized files
     /// > it created.
     metadata_directory: Option<PathBuf>,
+    /// Package id such as `foo-1.2.3`, for error reporting
+    package_id: String,
 }
 
 impl SourceBuild {
     /// Create a virtual environment in which to build a source distribution, extracting the
     /// contents from an archive if necessary.
+    ///
+    /// `package_id` is for error reporting only.
     pub async fn setup(
         source: &Path,
         subdirectory: Option<&Path>,
         interpreter_info: &InterpreterInfo,
         build_context: &impl BuildContext,
         source_build_context: SourceBuildContext,
+        package_id: &str,
     ) -> Result<SourceBuild, Error> {
         let temp_dir = tempdir()?;
 
@@ -232,8 +296,14 @@ impl SourceBuild {
         };
 
         if let Some(pep517_backend) = &pep517_backend {
-            create_pep517_build_environment(&source_tree, &venv, pep517_backend, build_context)
-                .await?;
+            create_pep517_build_environment(
+                &source_tree,
+                &venv,
+                pep517_backend,
+                build_context,
+                package_id,
+            )
+            .await?;
         } else {
             if !source_tree.join("setup.py").is_file() {
                 return Err(Error::InvalidSourceDistribution(
@@ -249,6 +319,7 @@ impl SourceBuild {
             pep517_backend,
             venv,
             metadata_directory: None,
+            package_id: package_id.to_string(),
         })
     }
 
@@ -284,7 +355,7 @@ impl SourceBuild {
             return Err(Error::from_command_output(
                 "Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(),
                 &output,
-            ));
+            &self.package_id));
         }
         let message = output
             .stdout
@@ -301,6 +372,7 @@ impl SourceBuild {
                         `prepare_metadata_for_build_wheel`: {err}"
                     ),
                     &output,
+                    &self.package_id,
                 )
             })?;
         if message.is_empty() {
@@ -336,6 +408,7 @@ impl SourceBuild {
                 return Err(Error::from_command_output(
                     "Failed building wheel through setup.py".to_string(),
                     &output,
+                    &self.package_id,
                 ));
             }
             let dist = fs::read_dir(self.source_tree.join("dist"))?;
@@ -346,7 +419,7 @@ impl SourceBuild {
                         "Expected exactly wheel in `dist/` after invoking setup.py, found {dist_dir:?}"
                     ),
                     &output,
-                ));
+                &self.package_id));
             };
             // TODO(konstin): Faster copy such as reflink? Or maybe don't really let the user pick the target dir
             let wheel = wheel_dir.join(dist_wheel.file_name());
@@ -382,6 +455,7 @@ impl SourceBuild {
             return Err(Error::from_command_output(
                 "Build backend failed to build wheel through `build_wheel()` ".to_string(),
                 &output,
+                &self.package_id,
             ));
         }
         let stdout = String::from_utf8_lossy(&output.stdout);
@@ -393,6 +467,7 @@ impl SourceBuild {
                 "Build backend did not return the wheel filename through `build_wheel()`"
                     .to_string(),
                 &output,
+                &self.package_id,
             ));
         };
         Ok(distribution_filename.to_string())
@@ -411,6 +486,7 @@ async fn create_pep517_build_environment(
     venv: &Virtualenv,
     pep517_backend: &Pep517Backend,
     build_context: &impl BuildContext,
+    package_id: &str,
 ) -> Result<(), Error> {
     debug!(
         "Calling `{}.get_requires_for_build_wheel()`",
@@ -438,6 +514,7 @@ async fn create_pep517_build_environment(
             "Build backend failed to determine extras requires with `get_requires_for_build_wheel`"
                 .to_string(),
             &output,
+            package_id,
         ));
     }
     let extra_requires = output
@@ -456,6 +533,7 @@ async fn create_pep517_build_environment(
                         `get_requires_for_build_wheel`: {err}"
             ),
             &output,
+            package_id,
         )
     })?;
     // Some packages (such as tqdm 4.66.1) list only extra requires that have already been part of
@@ -541,3 +619,69 @@ fn run_python_script(
         .output()
         .map_err(|err| Error::CommandFailed(python_interpreter.to_path_buf(), err))
 }
+
+#[cfg(test)]
+mod test {
+    use std::process::{ExitStatus, Output};
+
+    use crate::Error;
+    use indoc::indoc;
+
+    #[test]
+    fn missing_header() {
+        let output = Output {
+            status: ExitStatus::default(), // This is wrong but `from_raw` is platform gated
+            stdout: indoc!(r#"
+                running bdist_wheel
+                running build
+                [...]
+                creating build/temp.linux-x86_64-cpython-39/pygraphviz
+                gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -DOPENSSL_NO_SSL3 -fPIC -DSWIG_PYTHON_STRICT_BYTE_CHAR -I/tmp/.tmpy6vVes/.venv/include -I/home/konsti/.pyenv/versions/3.9.18/include/python3.9 -c pygraphviz/graphviz_wrap.c -o build/temp.linux-x86_64-cpython-39/pygraphviz/graphviz_wrap.o
+                "#
+            ).as_bytes().to_vec(),
+            stderr: indoc!(r#"
+                warning: no files found matching '*.png' under directory 'doc'
+                warning: no files found matching '*.txt' under directory 'doc'
+                [...]
+                no previously-included directories found matching 'doc/build'
+                pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory
+                 3020 | #include "graphviz/cgraph.h"
+                      |          ^~~~~~~~~~~~~~~~~~~
+                compilation terminated.
+                error: command '/usr/bin/gcc' failed with exit code 1
+                "#
+            ).as_bytes().to_vec(),
+        };
+
+        let err = Error::from_command_output(
+            "Failed building wheel through setup.py".to_string(),
+            &output,
+            "pygraphviz-1.11",
+        );
+        assert!(matches!(err, Error::MissingHeader { .. }));
+        insta::assert_display_snapshot!(err, @r###"
+        Failed building wheel through setup.py:
+        --- stdout:
+        running bdist_wheel
+        running build
+        [...]
+        creating build/temp.linux-x86_64-cpython-39/pygraphviz
+        gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -DOPENSSL_NO_SSL3 -fPIC -DSWIG_PYTHON_STRICT_BYTE_CHAR -I/tmp/.tmpy6vVes/.venv/include -I/home/konsti/.pyenv/versions/3.9.18/include/python3.9 -c pygraphviz/graphviz_wrap.c -o build/temp.linux-x86_64-cpython-39/pygraphviz/graphviz_wrap.o
+        --- stderr:
+        warning: no files found matching '*.png' under directory 'doc'
+        warning: no files found matching '*.txt' under directory 'doc'
+        [...]
+        no previously-included directories found matching 'doc/build'
+        pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory
+         3020 | #include "graphviz/cgraph.h"
+              |          ^~~~~~~~~~~~~~~~~~~
+        compilation terminated.
+        error: command '/usr/bin/gcc' failed with exit code 1
+        ---
+        "###);
+        insta::assert_display_snapshot!(
+            std::error::Error::source(&err).unwrap(),
+            @r###"This error likely indicates that you need to install a library that provides "graphviz/cgraph.h" for pygraphviz-1.11"###
+        );
+    }
+}

crates/puffin-dev/src/build.rs

@@ -54,7 +54,13 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
         false,
     );
     let wheel = build_dispatch
-        .build_source(&args.sdist, args.subdirectory.as_deref(), &wheel_dir)
+        .build_source(
+            &args.sdist,
+            args.subdirectory.as_deref(),
+            &wheel_dir,
+            // Good enough for the dev command
+            &args.sdist.display().to_string(),
+        )
         .await?;
     Ok(wheel_dir.join(wheel))
 }

crates/puffin-dispatch/src/lib.rs

@@ -91,9 +91,12 @@ impl BuildContext for BuildDispatch {
                 &self.client,
                 self,
             );
-            let resolution_graph = resolver.resolve().await.context(
-                "No solution found when resolving build dependencies for source distribution:",
-            )?;
+            let resolution_graph = resolver.resolve().await.with_context(|| {
+                format!(
+                    "No solution found when resolving build dependencies for source distribution: {}",
+                    requirements.iter().map(ToString::to_string).join(", "),
+                )
+            })?;
             Ok(resolution_graph.requirements())
         })
     }
@@ -236,6 +239,7 @@ impl BuildContext for BuildDispatch {
         source: &'a Path,
         subdirectory: Option<&'a Path>,
         wheel_dir: &'a Path,
+        package_id: &'a str,
     ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'a>> {
         Box::pin(async move {
             if self.no_build {
@@ -247,6 +251,7 @@ impl BuildContext for BuildDispatch {
                 &self.interpreter_info,
                 self,
                 self.source_build_context.clone(),
+                package_id,
             )
             .await?;
             Ok(builder.build(wheel_dir)?)

crates/puffin-installer/src/builder.rs

@@ -90,6 +90,7 @@ async fn build_sdist<T: BuildContext + Send + Sync>(
             &distribution.sdist_file,
             distribution.subdirectory.as_deref(),
             &wheel_dir,
+            &distribution.remote.to_string(),
         )
         .await?;
     let wheel_filename = wheel_dir.join(disk_filename);

crates/puffin-resolver/src/distribution/source_distribution.rs

@@ -144,7 +144,12 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> {
         // Build the wheel.
         let disk_filename = self
             .build_context
-            .build_source(&sdist_file, subdirectory.as_deref(), &wheel_dir)
+            .build_source(
+                &sdist_file,
+                subdirectory.as_deref(),
+                &wheel_dir,
+                &distribution.to_string(),
+            )
             .await?;
 
         // Read the metadata from the wheel.

crates/puffin-resolver/tests/resolver.rs

@@ -54,6 +54,7 @@ impl BuildContext for DummyContext {
         _sdist: &'a Path,
         _subdirectory: Option<&'a Path>,
         _wheel_dir: &'a Path,
+        _package_id: &'a str,
     ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'a>> {
         panic!("The test should not need to build source distributions")
     }

crates/puffin-traits/src/lib.rs

@@ -81,10 +81,13 @@ pub trait BuildContext {
     /// Build a source distribution into a wheel from an archive.
     ///
     /// Returns the filename of the built wheel inside the given `wheel_dir`.
+    ///
+    /// `package_id` is for error reporting only.
     fn build_source<'a>(
         &'a self,
         source: &'a Path,
         subdirectory: Option<&'a Path>,
         wheel_dir: &'a Path,
+        package_id: &'a str,
     ) -> Pin<Box<dyn Future<Output = anyhow::Result<String>> + Send + 'a>>;
 }