Add assertions of expected scenario results (#791)

Uses new metadata added in https://github.com/zanieb/packse/pull/61 to
assert that resolution succeeded or failed _and_ that the installed
package versions match the expected result.

crates/puffin-cli/tests/pip_install_scenarios.rs

@@ -4,9 +4,12 @@
 ///
 /// GENERATED WITH `./scripts/scenarios/update.py`
 /// SCENARIOS FROM `https://github.com/zanieb/packse/tree/d899bfe2c3c33fcb9ba5eac0162236a8e8d8cbcf/scenarios`
+use std::path::Path;
 use std::process::Command;
 
 use anyhow::Result;
+use assert_cmd::assert::Assert;
+use assert_cmd::prelude::*;
 use insta_cmd::_macro_support::insta;
 use insta_cmd::{assert_cmd_snapshot, get_cargo_bin};
 
@@ -14,6 +17,28 @@ use common::{create_venv, BIN_NAME, INSTA_FILTERS};
 
 mod common;
 
+fn assert_command(venv: &Path, command: &str, temp_dir: &Path) -> Assert {
+    Command::new(venv.join("bin").join("python"))
+        .arg("-c")
+        .arg(command)
+        .current_dir(temp_dir)
+        .assert()
+}
+
+fn assert_installed(venv: &Path, package: &'static str, version: &'static str, temp_dir: &Path) {
+    assert_command(
+        venv,
+        format!("import {package} as package; print(package.__version__, end='')").as_str(),
+        temp_dir,
+    )
+    .success()
+    .stdout(version);
+}
+
+fn assert_not_installed(venv: &Path, package: &'static str, temp_dir: &Path) {
+    assert_command(venv, format!("import {package}").as_str(), temp_dir).failure();
+}
+
 /// requires-package-only-prereleases
 ///
 /// The user requires any version of package `a` which only has pre-release versions
@@ -63,6 +88,15 @@ fn requires_package_only_prereleases() -> Result<()> {
         "###);
     });
 
+    // Since there are only pre-release versions of `a` available, it should be
+    // installed even though the user did not include a pre-release specifier.
+    assert_installed(
+        &venv,
+        "requires_package_only_prereleases_5829a64d_a",
+        "1.0.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -115,6 +149,14 @@ fn requires_package_only_prereleases_in_range() -> Result<()> {
         "###);
     });
 
+    // Since there are stable versions of `a` available, pre-release versions should
+    // not be selected without explicit opt-in.
+    assert_not_installed(
+        &venv,
+        "requires_package_only_prereleases_in_range_2b0594c8_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -154,6 +196,7 @@ fn requires_package_only_prereleases_in_range_global_opt_in() -> Result<()> {
         assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
             .arg("pip-install")
             .arg("requires-package-only-prereleases-in-range-global-opt-in-51f94da2-a>0.1.0")
+            .arg("--prerelease=allow")
             .arg("--extra-index-url")
             .arg("https://test.pypi.org/simple")
             .arg("--cache-dir")
@@ -161,16 +204,25 @@ fn requires_package_only_prereleases_in_range_global_opt_in() -> Result<()> {
             .env("VIRTUAL_ENV", venv.as_os_str())
             .env("PUFFIN_NO_WRAP", "1")
             .current_dir(&temp_dir), @r###"
-        success: false
+        success: true
-        exit_code: 1
+        exit_code: 0
         ----- stdout -----
 
         ----- stderr -----
-          × No solution found when resolving dependencies:
+        Resolved 1 package in [TIME]
-          ╰─▶ Because there is no version of a available matching >0.1.0 and root depends on a>0.1.0, version solving failed.
+        Downloaded 1 package in [TIME]
+        Installed 1 package in [TIME]
+         + a==1.0.0a1
         "###);
     });
 
+    assert_installed(
+        &venv,
+        "requires_package_only_prereleases_in_range_global_opt_in_51f94da2_a",
+        "1.0.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -225,6 +277,15 @@ fn requires_package_prerelease_and_final_any() -> Result<()> {
         "###);
     });
 
+    // Since the user did not provide a pre-release specifier, the older stable version
+    // should be selected.
+    assert_installed(
+        &venv,
+        "requires_package_prerelease_and_final_any_66989e88_a",
+        "0.1.0",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -286,6 +347,14 @@ fn requires_package_prerelease_specified_only_final_available() -> Result<()> {
         "###);
     });
 
+    // The latest stable version should be selected.
+    assert_installed(
+        &venv,
+        "requires_package_prerelease_specified_only_final_available_8c3e26d4_a",
+        "0.3.0",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -347,6 +416,14 @@ fn requires_package_prerelease_specified_only_prerelease_available() -> Result<(
         "###);
     });
 
+    // The latest pre-release version should be selected.
+    assert_installed(
+        &venv,
+        "requires_package_prerelease_specified_only_prerelease_available_fa8a64e0_a",
+        "0.3.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -411,6 +488,15 @@ fn requires_package_prerelease_specified_mixed_available() -> Result<()> {
         "###);
     });
 
+    // Since the user provided a pre-release specifier, the latest pre-release version
+    // should be selected.
+    assert_installed(
+        &venv,
+        "requires_package_prerelease_specified_mixed_available_caf5dd1a_a",
+        "1.0.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -469,6 +555,14 @@ fn requires_package_multiple_prereleases_kinds() -> Result<()> {
         "###);
     });
 
+    // Release candidates should be the highest precedence pre-release kind.
+    assert_installed(
+        &venv,
+        "requires_package_multiple_prereleases_kinds_08c2f99b_a",
+        "1.0.0rc1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -529,6 +623,14 @@ fn requires_package_multiple_prereleases_numbers() -> Result<()> {
         "###);
     });
 
+    // The latest alpha version should be selected.
+    assert_installed(
+        &venv,
+        "requires_package_multiple_prereleases_numbers_4cf7acef_a",
+        "1.0.0a3",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -590,6 +692,21 @@ fn requires_transitive_package_only_prereleases() -> Result<()> {
         "###);
     });
 
+    // Since there are only pre-release versions of `b` available, it should be
+    // selected even though the user did not opt-in to pre-releases.
+    assert_installed(
+        &venv,
+        "requires_transitive_package_only_prereleases_fa02005e_a",
+        "0.1.0",
+        &temp_dir,
+    );
+    assert_installed(
+        &venv,
+        "requires_transitive_package_only_prereleases_fa02005e_b",
+        "1.0.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -651,6 +768,15 @@ fn requires_transitive_package_only_prereleases_in_range() -> Result<()> {
         "###);
     });
 
+    // Since there are stable versions of `b` available, the pre-release version should
+    // not be selected without explicit opt-in. The available version is excluded by
+    // the range requested by the user.
+    assert_not_installed(
+        &venv,
+        "requires_transitive_package_only_prereleases_in_range_4800779d_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -718,6 +844,21 @@ fn requires_transitive_package_only_prereleases_in_range_opt_in() -> Result<()>
         "###);
     });
 
+    // Since the user included a dependency on `b` with a pre-release specifier, a pre-
+    // release version can be selected.
+    assert_installed(
+        &venv,
+        "requires_transitive_package_only_prereleases_in_range_opt_in_4ca10c42_a",
+        "0.1.0",
+        &temp_dir,
+    );
+    assert_installed(
+        &venv,
+        "requires_transitive_package_only_prereleases_in_range_opt_in_4ca10c42_b",
+        "1.0.0a1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -789,6 +930,18 @@ fn requires_transitive_prerelease_and_stable_dependency() -> Result<()> {
         "###);
     });
 
+    // Since the user did not explicitly opt-in to a prerelease, it cannot be selected.
+    assert_not_installed(
+        &venv,
+        "requires_transitive_prerelease_and_stable_dependency_31b546ef_a",
+        &temp_dir,
+    );
+    assert_not_installed(
+        &venv,
+        "requires_transitive_prerelease_and_stable_dependency_31b546ef_b",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -866,6 +1019,26 @@ fn requires_transitive_prerelease_and_stable_dependency_opt_in() -> Result<()> {
         "###);
     });
 
+    // Since the user explicitly opted-in to a prerelease for `c`, it can be installed.
+    assert_installed(
+        &venv,
+        "requires_transitive_prerelease_and_stable_dependency_opt_in_dd00a87f_a",
+        "1.0.0",
+        &temp_dir,
+    );
+    assert_installed(
+        &venv,
+        "requires_transitive_prerelease_and_stable_dependency_opt_in_dd00a87f_b",
+        "1.0.0",
+        &temp_dir,
+    );
+    assert_installed(
+        &venv,
+        "requires_transitive_prerelease_and_stable_dependency_opt_in_dd00a87f_c",
+        "2.0.0b1",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -911,6 +1084,12 @@ fn requires_package_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_package_does_not_exist_57cd4136_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -960,6 +1139,12 @@ fn requires_exact_version_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_exact_version_does_not_exist_eaa03067_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1012,6 +1197,12 @@ fn requires_greater_version_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_greater_version_does_not_exist_6e8e01df_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1066,6 +1257,12 @@ fn requires_less_version_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_less_version_does_not_exist_e45cec3c_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1116,6 +1313,12 @@ fn transitive_requires_package_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "transitive_requires_package_does_not_exist_aca2796a_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1170,6 +1373,17 @@ fn requires_direct_incompatible_versions() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_direct_incompatible_versions_063ec9d3_a",
+        &temp_dir,
+    );
+    assert_not_installed(
+        &venv,
+        "requires_direct_incompatible_versions_063ec9d3_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1234,6 +1448,17 @@ fn requires_transitive_incompatible_with_root_version() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_transitive_incompatible_with_root_version_638350f3_a",
+        &temp_dir,
+    );
+    assert_not_installed(
+        &venv,
+        "requires_transitive_incompatible_with_root_version_638350f3_b",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1304,6 +1529,17 @@ fn requires_transitive_incompatible_with_transitive() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_transitive_incompatible_with_transitive_9b595175_a",
+        &temp_dir,
+    );
+    assert_not_installed(
+        &venv,
+        "requires_transitive_incompatible_with_transitive_9b595175_b",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1354,6 +1590,12 @@ fn requires_python_version_does_not_exist() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_python_version_does_not_exist_0825b69c_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1405,6 +1647,12 @@ fn requires_python_version_less_than_current() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_python_version_less_than_current_f9296b84_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1459,6 +1707,12 @@ fn requires_python_version_greater_than_current() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_python_version_greater_than_current_a11d5394_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1534,6 +1788,12 @@ fn requires_python_version_greater_than_current_many() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_python_version_greater_than_current_many_02dc550c_a",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1598,6 +1858,13 @@ fn requires_python_version_greater_than_current_backtrack() -> Result<()> {
         "###);
     });
 
+    assert_installed(
+        &venv,
+        "requires_python_version_greater_than_current_backtrack_ef060cef_a",
+        "1.0.0",
+        &temp_dir,
+    );
+
     Ok(())
 }
 
@@ -1669,5 +1936,11 @@ fn requires_python_version_greater_than_current_excluded() -> Result<()> {
         "###);
     });
 
+    assert_not_installed(
+        &venv,
+        "requires_python_version_greater_than_current_excluded_1bde0c18_a",
+        &temp_dir,
+    );
+
     Ok(())
 }

scripts/scenarios/template.mustache

@@ -6,16 +6,48 @@
 /// GENERATED WITH `{{generated_with}}`
 /// SCENARIOS FROM `{{generated_from}}`
 
-
+use std::path::Path;
 use std::process::Command;
 
 use anyhow::Result;
+use assert_cmd::assert::Assert;
+use assert_cmd::prelude::*;
 use insta_cmd::_macro_support::insta;
 use insta_cmd::{assert_cmd_snapshot, get_cargo_bin};
 
 use common::{create_venv, BIN_NAME, INSTA_FILTERS};
 
 mod common;
+
+fn assert_command(venv: &Path, command: &str, temp_dir: &Path) -> Assert {
+    Command::new(venv.join("bin").join("python"))
+        .arg("-c")
+        .arg(command)
+        .current_dir(temp_dir)
+        .assert()
+}
+
+fn assert_installed(
+    venv: &Path,
+    package: &'static str,
+    version: &'static str,
+    temp_dir: &Path,
+) {
+    assert_command(
+        venv,
+        format!(
+            "import {package} as package; print(package.__version__, end='')"
+        )
+        .as_str(),
+        temp_dir,
+    )
+    .success()
+    .stdout(version);
+}
+
+fn assert_not_installed(venv: &Path, package: &'static str, temp_dir: &Path) {
+    assert_command(venv, format!("import {package}").as_str(), temp_dir).failure();
+}
 {{#scenarios}}
 
 /// {{name}}
@@ -46,6 +78,9 @@ fn {{normalized_name}}() -> Result<()> {
             {{#root.requires}}
             .arg("{{prefix}}-{{.}}")
             {{/root.requires}}
+            {{#environment.prereleases}}
+            .arg("--prerelease=allow")
+            {{/environment.prereleases}}
             .arg("--extra-index-url")
             .arg("https://test.pypi.org/simple")
             .arg("--cache-dir")
@@ -56,6 +91,25 @@ fn {{normalized_name}}() -> Result<()> {
         "###);
     });
 
+    {{#expected.explanation_lines}}
+    // {{.}}
+    {{/expected.explanation_lines}}
+    {{#expected.satisfiable}}
+    {{#expected.packages_list}}
+    assert_installed(
+        &venv,
+        "{{prefix_module}}_{{package_module}}",
+        "{{version}}",
+        &temp_dir
+    );
+    {{/expected.packages_list}}
+    {{/expected.satisfiable}}
+    {{^expected.satisfiable}}
+    {{#root.requires_packages}}
+    assert_not_installed(&venv, "{{prefix_module}}_{{package_module}}", &temp_dir);
+    {{/root.requires_packages}}
+    {{/expected.satisfiable}}
+
     Ok(())
 }
 {{/scenarios}}

scripts/scenarios/update.py

@@ -21,6 +21,7 @@ import shutil
 import subprocess
 import sys
 import textwrap
+import packaging.requirements
 from pathlib import Path
 
 
@@ -106,7 +107,7 @@ else:
     )
 
     if commit != PACKSE_COMMIT:
-        print("WARNING: Expected commit {PACKSE_COMMIT!r} but found {commit!r}.")
+        print(f"WARNING: Expected commit {PACKSE_COMMIT!r} but found {commit!r}.")
 
 print("Loading scenario metadata...", file=sys.stderr)
 data = json.loads(
@@ -137,6 +138,46 @@ for index, scenario in enumerate(data["scenarios"]):
 for scenario in data["scenarios"]:
     scenario["description_lines"] = textwrap.wrap(scenario["description"], width=80)
 
+
+# Wrap the expected explanation onto multiple lines
+for scenario in data["scenarios"]:
+    expected = scenario["expected"]
+    expected["explanation_lines"] = (
+        textwrap.wrap(expected["explanation"], width=80)
+        if expected["explanation"]
+        else []
+    )
+
+# Convert the expected packages into a list for rendering
+for scenario in data["scenarios"]:
+    expected = scenario["expected"]
+    expected["packages_list"] = []
+    for key, value in expected["packages"].items():
+        expected["packages_list"].append(
+            {
+                "package": key,
+                "version": value,
+                # Include a converted version of the package name to its Python module
+                "package_module": key.replace("-", "_"),
+            }
+        )
+
+
+# Convert the required packages into a list without versions
+for scenario in data["scenarios"]:
+    requires_packages = scenario["root"]["requires_packages"] = []
+    for requirement in scenario["root"]["requires"]:
+        package = packaging.requirements.Requirement(requirement).name
+        requires_packages.append(
+            {"package": package, "package_module": package.replace("-", "_")}
+        )
+
+
+# Include the Python module name of the prefix
+for scenario in data["scenarios"]:
+    scenario["prefix_module"] = scenario["prefix"].replace("-", "_")
+
+
 # Render the template
 print("Rendering template...", file=sys.stderr)
 output = chevron_blue.render(template=TEMPLATE.read_text(), data=data, no_escape=True)