Allow additional assertions on command output (#1226)

In the scenario tests, we want to make sure we're actually conforming to
the scenario's expectations, so we now have an extra assertion on
whether resolution failed or succeeded as well as that it includes the
given packages.

Closes #1112
Closes #1030

crates/puffin/tests/common/mod.rs

@@ -141,16 +141,16 @@ pub fn run_and_format(command: &mut std::process::Command) -> (String, Output) {
 #[allow(unused_macros)]
 macro_rules! puffin_snapshot {
     ($spawnable:expr, @$snapshot:literal) => {{
-        puffin_snapshot!($crate::common::INSTA_FILTERS.to_vec(), $spawnable, @$snapshot);
+        puffin_snapshot!($crate::common::INSTA_FILTERS.to_vec(), $spawnable, @$snapshot)
     }};
     ($filters:expr, $spawnable:expr, @$snapshot:literal) => {{
+        let (snapshot, output) = $crate::common::run_and_format($spawnable);
         ::insta::with_settings!({
             filters => $filters.to_vec()
         }, {
-            let (snapshot, output) = $crate::common::run_and_format($spawnable);
             ::insta::assert_snapshot!(snapshot, @$snapshot);
-            output
         });
+        output
     }};
 }
 

crates/puffin/tests/pip_compile_scenarios.rs

@@ -1,7 +1,7 @@
 //! DO NOT EDIT
 //!
 //! Generated with ./scripts/scenarios/update.py
-//! Scenarios from <https://github.com/zanieb/packse/tree/a451c95105db0c2897d0ad1bde0703ba0fc40d3a/scenarios>
+//! Scenarios from <https://github.com/zanieb/packse/tree/0563417be973397d05b45cd2c5b415d7215161e3/scenarios>
 //!
 #![cfg(all(feature = "python", feature = "pypi"))]
 
@@ -9,14 +9,15 @@ use std::path::PathBuf;
 use std::process::Command;
 
 use anyhow::Result;
+use assert_cmd::assert::OutputAssertExt;
 use assert_fs::fixture::{FileWriteStr, PathChild};
 #[cfg(unix)]
 use fs_err::os::unix::fs::symlink as symlink_file;
 #[cfg(windows)]
 use fs_err::os::windows::fs::symlink_file;
+use predicates::prelude::predicate;
 
-use crate::common::get_bin;
-use common::{puffin_snapshot, TestContext, INSTA_FILTERS};
+use common::{get_bin, puffin_snapshot, TestContext, INSTA_FILTERS};
 use puffin_interpreter::find_requested_python;
 
 mod common;
@@ -88,22 +89,27 @@ fn requires_incompatible_python_version_compatible_override() -> Result<()> {
     let requirements_in = context.temp_dir.child("requirements.in");
     requirements_in.write_str("a-006fed96==1.0.0")?;
 
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.11")
         , @r###"
-    success: true
-    exit_code: 0
-    ----- stdout -----
-    # This file was autogenerated by Puffin v[VERSION] via the following command:
-    #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.11
-    albatross==1.0.0
+                 success: true
+                 exit_code: 0
+                 ----- stdout -----
+                 # This file was autogenerated by Puffin v[VERSION] via the following command:
+                 #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.11
+                 albatross==1.0.0
 
-    ----- stderr -----
-    warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
-    Resolved 1 package in [TIME]
-    "###
+                 ----- stderr -----
+                 warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
+                 Resolved 1 package in [TIME]
+                 "###
     );
 
+    output
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("a-006fed96==1.0.0"));
+
     Ok(())
 }
 
@@ -136,21 +142,23 @@ fn requires_compatible_python_version_incompatible_override() -> Result<()> {
     let requirements_in = context.temp_dir.child("requirements.in");
     requirements_in.write_str("a-8c1b0389==1.0.0")?;
 
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.9")
         , @r###"
-    success: false
-    exit_code: 1
-    ----- stdout -----
+                 success: false
+                 exit_code: 1
+                 ----- stdout -----
 
-    ----- stderr -----
-    warning: The requested Python version 3.9 is not available; 3.11.7 will be used to build dependencies instead.
-      × No solution found when resolving dependencies:
-      ╰─▶ Because the requested Python version (3.9) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
-          And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
-    "###
+                 ----- stderr -----
+                 warning: The requested Python version 3.9 is not available; 3.11.7 will be used to build dependencies instead.
+                   × No solution found when resolving dependencies:
+                   ╰─▶ Because the requested Python version (3.9) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
+                       And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
+                 "###
     );
 
+    output.assert().failure();
+
     Ok(())
 }
 
@@ -187,21 +195,23 @@ fn requires_incompatible_python_version_compatible_override_no_wheels() -> Resul
     // Since there are no wheels for the package and it is not compatible with the
     // local installation, we cannot build the source distribution to determine its
     // dependencies.
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.11")
         , @r###"
-    success: false
-    exit_code: 1
-    ----- stdout -----
+                 success: false
+                 exit_code: 1
+                 ----- stdout -----
 
-    ----- stderr -----
-    warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
-      × No solution found when resolving dependencies:
-      ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
-          And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
-    "###
+                 ----- stderr -----
+                 warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
+                   × No solution found when resolving dependencies:
+                   ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
+                       And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
+                 "###
     );
 
+    output.assert().failure();
+
     Ok(())
 }
 
@@ -213,7 +223,7 @@ fn requires_incompatible_python_version_compatible_override_no_wheels() -> Resul
 /// version installed elsewhere on their system.
 ///
 /// ```text
-/// ae5b2665
+/// 23e00318
 /// ├── environment
 /// │   ├── python3.11
 /// │   └── python3.9 (active)
@@ -232,29 +242,34 @@ fn requires_incompatible_python_version_compatible_override_no_wheels_available_
 
     // In addition to the standard filters, swap out package names for more realistic messages
     let mut filters = INSTA_FILTERS.to_vec();
-    filters.push((r"a-ae5b2665", "albatross"));
-    filters.push((r"-ae5b2665", ""));
+    filters.push((r"a-23e00318", "albatross"));
+    filters.push((r"-23e00318", ""));
 
     let requirements_in = context.temp_dir.child("requirements.in");
-    requirements_in.write_str("a-ae5b2665==1.0.0")?;
+    requirements_in.write_str("a-23e00318==1.0.0")?;
 
     // Since there is a compatible Python version available on the system, it should be
     // used to build the source distributions.
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.11")
         , @r###"
-    success: true
-    exit_code: 0
-    ----- stdout -----
-    # This file was autogenerated by Puffin v[VERSION] via the following command:
-    #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.11
-    albatross==1.0.0
+                 success: true
+                 exit_code: 0
+                 ----- stdout -----
+                 # This file was autogenerated by Puffin v[VERSION] via the following command:
+                 #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.11
+                 albatross==1.0.0
 
-    ----- stderr -----
-    Resolved 1 package in [TIME]
-    "###
+                 ----- stderr -----
+                 Resolved 1 package in [TIME]
+                 "###
     );
 
+    output
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("a-23e00318==1.0.0"));
+
     Ok(())
 }
 
@@ -291,21 +306,23 @@ fn requires_incompatible_python_version_compatible_override_no_compatible_wheels
     // Since there are no compatible wheels for the package and it is not compatible
     // with the local installation, we cannot build the source distribution to
     // determine its dependencies.
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.11")
         , @r###"
-    success: false
-    exit_code: 1
-    ----- stdout -----
+                 success: false
+                 exit_code: 1
+                 ----- stdout -----
 
-    ----- stderr -----
-    warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
-      × No solution found when resolving dependencies:
-      ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
-          And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
-    "###
+                 ----- stderr -----
+                 warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
+                   × No solution found when resolving dependencies:
+                   ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
+                       And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
+                 "###
     );
 
+    output.assert().failure();
+
     Ok(())
 }
 
@@ -347,29 +364,31 @@ fn requires_incompatible_python_version_compatible_override_other_wheel() -> Res
     // target and it is not compatible with the local installation, we cannot build the
     // source distribution to determine its dependencies. The other version has wheels
     // available, but is not compatible with the target version and cannot be used.
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.11")
         , @r###"
-    success: false
-    exit_code: 1
-    ----- stdout -----
+                 success: false
+                 exit_code: 1
+                 ----- stdout -----
 
-    ----- stderr -----
-    warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
-      × No solution found when resolving dependencies:
-      ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
-          And because there are no versions of albatross that satisfy any of:
-              albatross<1.0.0
-              albatross>1.0.0,<2.0.0
-              albatross>2.0.0
-          we can conclude that albatross<2.0.0 cannot be used. (1)
+                 ----- stderr -----
+                 warning: The requested Python version 3.11 is not available; 3.9.18 will be used to build dependencies instead.
+                   × No solution found when resolving dependencies:
+                   ╰─▶ Because the current Python version (3.9.18) does not satisfy Python>=3.10 and albatross==1.0.0 depends on Python>=3.10, we can conclude that albatross==1.0.0 cannot be used.
+                       And because there are no versions of albatross that satisfy any of:
+                           albatross<1.0.0
+                           albatross>1.0.0,<2.0.0
+                           albatross>2.0.0
+                       we can conclude that albatross<2.0.0 cannot be used. (1)
 
-          Because the requested Python version (3.11) does not satisfy Python>=3.12 and albatross==2.0.0 depends on Python>=3.12, we can conclude that albatross==2.0.0 cannot be used.
-          And because we know from (1) that albatross<2.0.0 cannot be used, we can conclude that all versions of albatross cannot be used.
-          And because you require albatross, we can conclude that the requirements are unsatisfiable.
-    "###
+                       Because the requested Python version (3.11) does not satisfy Python>=3.12 and albatross==2.0.0 depends on Python>=3.12, we can conclude that albatross==2.0.0 cannot be used.
+                       And because we know from (1) that albatross<2.0.0 cannot be used, we can conclude that all versions of albatross cannot be used.
+                       And because you require albatross, we can conclude that the requirements are unsatisfiable.
+                 "###
     );
 
+    output.assert().failure();
+
     Ok(())
 }
 
@@ -404,20 +423,22 @@ fn requires_python_patch_version_override_no_patch() -> Result<()> {
 
     // Since the resolver is asked to solve with 3.8, the minimum compatible Python
     // requirement is treated as 3.8.0.
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.8")
         , @r###"
-    success: false
-    exit_code: 1
-    ----- stdout -----
+                 success: false
+                 exit_code: 1
+                 ----- stdout -----
 
-    ----- stderr -----
-      × No solution found when resolving dependencies:
-      ╰─▶ Because the requested Python version (3.8) does not satisfy Python>=3.8.4 and albatross==1.0.0 depends on Python>=3.8.4, we can conclude that albatross==1.0.0 cannot be used.
-          And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
-    "###
+                 ----- stderr -----
+                   × No solution found when resolving dependencies:
+                   ╰─▶ Because the requested Python version (3.8) does not satisfy Python>=3.8.4 and albatross==1.0.0 depends on Python>=3.8.4, we can conclude that albatross==1.0.0 cannot be used.
+                       And because you require albatross==1.0.0, we can conclude that the requirements are unsatisfiable.
+                 "###
     );
 
+    output.assert().failure();
+
     Ok(())
 }
 
@@ -450,21 +471,26 @@ fn requires_python_patch_version_override_patch_compatible() -> Result<()> {
     let requirements_in = context.temp_dir.child("requirements.in");
     requirements_in.write_str("a-844899bd==1.0.0")?;
 
-    puffin_snapshot!(filters, command(&context, python_versions)
+    let output = puffin_snapshot!(filters, command(&context, python_versions)
         .arg("--python-version=3.8.0")
         , @r###"
-    success: true
-    exit_code: 0
-    ----- stdout -----
-    # This file was autogenerated by Puffin v[VERSION] via the following command:
-    #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.8.0
-    albatross==1.0.0
+                 success: true
+                 exit_code: 0
+                 ----- stdout -----
+                 # This file was autogenerated by Puffin v[VERSION] via the following command:
+                 #    puffin pip compile requirements.in --extra-index-url https://test.pypi.org/simple --cache-dir [CACHE_DIR] --python-version=3.8.0
+                 albatross==1.0.0
 
-    ----- stderr -----
-    warning: The requested Python version 3.8.0 is not available; 3.8.18 will be used to build dependencies instead.
-    Resolved 1 package in [TIME]
-    "###
+                 ----- stderr -----
+                 warning: The requested Python version 3.8.0 is not available; 3.8.18 will be used to build dependencies instead.
+                 Resolved 1 package in [TIME]
+                 "###
     );
 
+    output
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("a-844899bd==1.0.0"));
+
     Ok(())
 }

crates/puffin/tests/pip_install_scenarios.rs

@@ -1,7 +1,7 @@
 //! DO NOT EDIT
 //!
 //! Generated with ./scripts/scenarios/update.py
-//! Scenarios from <https://github.com/zanieb/packse/tree/a451c95105db0c2897d0ad1bde0703ba0fc40d3a/scenarios>
+//! Scenarios from <https://github.com/zanieb/packse/tree/0563417be973397d05b45cd2c5b415d7215161e3/scenarios>
 //!
 #![cfg(all(feature = "python", feature = "pypi"))]