Expand environment variables in URLs (#640)

## Summary

This PR enables users to express relative dependencies via environment
variables. Like pip, PDM, Hatch, Rye, and others, we now allow users to
express dependencies like:

```text
flask @ file://${PROJECT_ROOT}/flask-3.0.0-py3-none-any.whl
```

In the compiled requirements file, we'll also preserve the unexpanded
environment variable.

Closes https://github.com/astral-sh/puffin/issues/592.

crates/pep508-rs/src/verbatim_url.rs

@@ -1,4 +1,8 @@
+use std::borrow::Cow;
 use std::ops::Deref;
+
+use once_cell::sync::Lazy;
+use regex::Regex;
 use url::Url;
 
 /// A wrapper around [`Url`] that preserves the original string.
@@ -14,9 +18,9 @@ pub struct VerbatimUrl {
 }
 
 impl VerbatimUrl {
-    /// Parse a URL from a string.
+    /// Parse a URL from a string, expanding any environment variables.
     pub fn parse(given: String) -> Result<Self, Error> {
-        let url = Url::parse(&given)?;
+        let url = Url::parse(&expand_env_vars(&given))?;
         Ok(Self {
             given: Some(given),
             url,
@@ -73,3 +77,38 @@ pub enum Error {
     #[error(transparent)]
     Url(#[from] url::ParseError),
 }
+
+/// Expand all available environment variables.
+///
+/// This is modeled off of pip's environment variable expansion, which states:
+///
+///   The only allowed format for environment variables defined in the
+///   requirement file is `${MY_VARIABLE_1}` to ensure two things:
+///
+///   1. Strings that contain a `$` aren't accidentally (partially) expanded.
+///   2. Ensure consistency across platforms for requirement files.
+///
+///   ...
+///
+///   Valid characters in variable names follow the `POSIX standard
+///   <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
+///   to uppercase letter, digits and the `_` (underscore).
+fn expand_env_vars(s: &str) -> Cow<'_, str> {
+    // Generate a URL-escaped project root, to be used via the `${PROJECT_ROOT}`
+    // environment variable. Ensure that it's URL-escaped.
+    static PROJECT_ROOT_FRAGMENT: Lazy<String> = Lazy::new(|| {
+        let project_root = std::env::current_dir().unwrap();
+        project_root.to_string_lossy().replace(' ', "%20")
+    });
+
+    static RE: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"(?P<var>\$\{(?P<name>[A-Z0-9_]+)})").unwrap());
+
+    RE.replace_all(s, |caps: &regex::Captures<'_>| {
+        let name = caps.name("name").unwrap().as_str();
+        std::env::var(name).unwrap_or_else(|_| match name {
+            "PROJECT_ROOT" => PROJECT_ROOT_FRAGMENT.clone(),
+            _ => caps["var"].to_owned(),
+        })
+    })
+}

crates/puffin-cli/tests/pip_compile.rs

@@ -2490,3 +2490,61 @@ fn preserve_url() -> Result<()> {
 
     Ok(())
 }
+
+/// Resolve a dependency from a URL, preserving the unexpanded environment variable as specified in
+/// the requirements file.
+#[test]
+fn preserve_env_var() -> Result<()> {
+    let temp_dir = TempDir::new()?;
+    let cache_dir = TempDir::new()?;
+    let venv = create_venv_py312(&temp_dir, &cache_dir);
+
+    // Download a wheel.
+    let response = reqwest::blocking::get("https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl")?;
+    let flask_wheel = temp_dir.child("flask-3.0.0-py3-none-any.whl");
+    let mut flask_wheel_file = std::fs::File::create(flask_wheel)?;
+    std::io::copy(&mut response.bytes()?.as_ref(), &mut flask_wheel_file)?;
+
+    let requirements_in = temp_dir.child("requirements.in");
+    requirements_in.write_str("flask @ file://${PROJECT_ROOT}/flask-3.0.0-py3-none-any.whl")?;
+
+    insta::with_settings!({
+        filters => INSTA_FILTERS.to_vec()
+    }, {
+        assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+            .arg("pip-compile")
+            .arg("requirements.in")
+            .arg("--cache-dir")
+            .arg(cache_dir.path())
+            .arg("--exclude-newer")
+            .arg(EXCLUDE_NEWER)
+            .env("VIRTUAL_ENV", venv.as_os_str())
+            .current_dir(&temp_dir), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+        # This file was autogenerated by Puffin v0.0.1 via the following command:
+        #    puffin pip-compile requirements.in --cache-dir [CACHE_DIR]
+        blinker==1.7.0
+            # via flask
+        click==8.1.7
+            # via flask
+        flask @ file://${PROJECT_ROOT}/flask-3.0.0-py3-none-any.whl
+        itsdangerous==2.1.2
+            # via flask
+        jinja2==3.1.2
+            # via flask
+        markupsafe==2.1.3
+            # via
+            #   jinja2
+            #   werkzeug
+        werkzeug==3.0.1
+            # via flask
+
+        ----- stderr -----
+        Resolved 7 packages in [TIME]
+        "###);
+    });
+
+    Ok(())
+}