Support environment variables in index URLs in requirements files (#2036)

## Summary

This also preserves the environment variables in the output file, e.g.:

```
Resolved 1 package in 216ms
# This file was autogenerated by uv via the following command:
#    uv pip compile requirements.in --emit-index-url
--index-url https://test.pypi.org/${SUFFIX}

requests==2.5.4.1
```

I'm torn on whether that's correct or undesirable here.

Closes #2035.

crates/distribution-types/src/index_url.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt::{Display, Formatter};
 use std::ops::Deref;
 use std::path::PathBuf;
@@ -8,39 +9,59 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use pep508_rs::{split_scheme, Scheme};
+use pep508_rs::{split_scheme, Scheme, VerbatimUrl};
 use uv_fs::normalize_url_path;
 
+use crate::Verbatim;
+
 static PYPI_URL: Lazy<Url> = Lazy::new(|| Url::parse("https://pypi.org/simple").unwrap());
 
+static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
+    Lazy::new(|| IndexUrl::Pypi(VerbatimUrl::from_url(PYPI_URL.clone())));
+
 /// The url of an index, newtype'd to avoid mixing it with file urls.
 #[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)]
 pub enum IndexUrl {
-    Pypi,
-    Url(Url),
+    Pypi(VerbatimUrl),
+    Url(VerbatimUrl),
 }
 
 impl Display for IndexUrl {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            Self::Pypi => Display::fmt(&*PYPI_URL, f),
+            Self::Pypi(url) => Display::fmt(url, f),
             Self::Url(url) => Display::fmt(url, f),
         }
     }
 }
 
+impl Verbatim for IndexUrl {
+    fn verbatim(&self) -> Cow<'_, str> {
+        match self {
+            Self::Pypi(url) => url.verbatim(),
+            Self::Url(url) => url.verbatim(),
+        }
+    }
+}
+
 impl FromStr for IndexUrl {
     type Err = url::ParseError;
 
-    fn from_str(url: &str) -> Result<Self, Self::Err> {
-        Ok(Self::from(Url::parse(url)?))
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let url = Url::parse(s)?;
+        let url = VerbatimUrl::from_url(url).with_given(s.to_owned());
+        if *url.raw() == *PYPI_URL {
+            Ok(Self::Pypi(url))
+        } else {
+            Ok(Self::Url(url))
+        }
     }
 }
 
-impl From<Url> for IndexUrl {
-    fn from(url: Url) -> Self {
-        if url == *PYPI_URL {
-            Self::Pypi
+impl From<VerbatimUrl> for IndexUrl {
+    fn from(url: VerbatimUrl) -> Self {
+        if *url.raw() == *PYPI_URL {
+            Self::Pypi(url)
         } else {
             Self::Url(url)
         }
@@ -50,8 +71,8 @@ impl From<Url> for IndexUrl {
 impl From<IndexUrl> for Url {
     fn from(index: IndexUrl) -> Self {
         match index {
-            IndexUrl::Pypi => PYPI_URL.clone(),
-            IndexUrl::Url(url) => url,
+            IndexUrl::Pypi(url) => url.to_url(),
+            IndexUrl::Url(url) => url.to_url(),
         }
     }
 }
@@ -61,7 +82,7 @@ impl Deref for IndexUrl {
 
     fn deref(&self) -> &Self::Target {
         match &self {
-            Self::Pypi => &PYPI_URL,
+            Self::Pypi(url) => url,
             Self::Url(url) => url,
         }
     }
@@ -152,7 +173,7 @@ impl Default for IndexLocations {
     /// By default, use the `PyPI` index.
     fn default() -> Self {
         Self {
-            index: Some(IndexUrl::Pypi),
+            index: Some(DEFAULT_INDEX_URL.clone()),
             extra_index: Vec::new(),
             flat_index: Vec::new(),
             no_index: false,
@@ -211,7 +232,7 @@ impl<'a> IndexLocations {
         } else {
             match self.index.as_ref() {
                 Some(index) => Some(index),
-                None => Some(&IndexUrl::Pypi),
+                None => Some(&DEFAULT_INDEX_URL),
             }
         }
     }
@@ -259,7 +280,7 @@ impl Default for IndexUrls {
     /// By default, use the `PyPI` index.
     fn default() -> Self {
         Self {
-            index: Some(IndexUrl::Pypi),
+            index: Some(DEFAULT_INDEX_URL.clone()),
             extra_index: Vec::new(),
             no_index: false,
         }
@@ -278,7 +299,7 @@ impl<'a> IndexUrls {
         } else {
             match self.index.as_ref() {
                 Some(index) => Some(index),
-                None => Some(&IndexUrl::Pypi),
+                None => Some(&DEFAULT_INDEX_URL),
             }
         }
     }

crates/pep508-rs/src/lib.rs

@@ -755,11 +755,11 @@ fn preprocess_url(
                 #[cfg(feature = "non-pep508-extensions")]
                 if let Some(working_dir) = working_dir {
                     return Ok(
-                        VerbatimUrl::from_path(path, working_dir).with_given(url.to_string())
+                        VerbatimUrl::parse_path(path, working_dir).with_given(url.to_string())
                     );
                 }
 
-                Ok(VerbatimUrl::from_absolute_path(path)
+                Ok(VerbatimUrl::parse_absolute_path(path)
                     .map_err(|err| Pep508Error {
                         message: Pep508ErrorSource::UrlError(err),
                         start,
@@ -783,10 +783,12 @@ fn preprocess_url(
             _ => {
                 #[cfg(feature = "non-pep508-extensions")]
                 if let Some(working_dir) = working_dir {
-                    return Ok(VerbatimUrl::from_path(url, working_dir).with_given(url.to_string()));
+                    return Ok(
+                        VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string())
+                    );
                 }
 
-                Ok(VerbatimUrl::from_absolute_path(url)
+                Ok(VerbatimUrl::parse_absolute_path(url)
                     .map_err(|err| Pep508Error {
                         message: Pep508ErrorSource::UrlError(err),
                         start,
@@ -800,10 +802,10 @@ fn preprocess_url(
         // Ex) `../editable/`
         #[cfg(feature = "non-pep508-extensions")]
         if let Some(working_dir) = working_dir {
-            return Ok(VerbatimUrl::from_path(url, working_dir).with_given(url.to_string()));
+            return Ok(VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string()));
         }
 
-        Ok(VerbatimUrl::from_absolute_path(url)
+        Ok(VerbatimUrl::parse_absolute_path(url)
             .map_err(|err| Pep508Error {
                 message: Pep508ErrorSource::UrlError(err),
                 start,

crates/pep508-rs/src/verbatim_url.rs

@@ -5,7 +5,7 @@ use std::path::{Component, Path, PathBuf};
 
 use once_cell::sync::Lazy;
 use regex::Regex;
-use url::Url;
+use url::{ParseError, Url};
 
 /// A wrapper around [`Url`] that preserves the original string.
 #[derive(Debug, Clone, Eq, derivative::Derivative)]
@@ -29,15 +29,26 @@ pub struct VerbatimUrl {
 
 impl VerbatimUrl {
     /// Parse a URL from a string, expanding any environment variables.
-    pub fn parse(given: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
-        let url = Url::parse(&expand_env_vars(given.as_ref(), true))
-            .map_err(|err| VerbatimUrlError::Url(given.as_ref().to_owned(), err))?;
+    pub fn parse(given: impl AsRef<str>) -> Result<Self, ParseError> {
+        let url = Url::parse(&expand_env_vars(given.as_ref(), true))?;
         Ok(Self { url, given: None })
     }
 
+    /// Create a [`VerbatimUrl`] from a [`Url`].
+    pub fn from_url(url: Url) -> Self {
+        Self { url, given: None }
+    }
+
+    /// Create a [`VerbatimUrl`] from a file path.
+    pub fn from_path(path: impl AsRef<Path>) -> Self {
+        let path = normalize_path(path.as_ref());
+        let url = Url::from_file_path(path).expect("path is absolute");
+        Self { url, given: None }
+    }
+
     /// Parse a URL from an absolute or relative path.
     #[cfg(feature = "non-pep508-extensions")] // PEP 508 arguably only allows absolute file URLs.
-    pub fn from_path(path: impl AsRef<str>, working_dir: impl AsRef<Path>) -> Self {
+    pub fn parse_path(path: impl AsRef<str>, working_dir: impl AsRef<Path>) -> Self {
         // Expand any environment variables.
         let path = PathBuf::from(expand_env_vars(path.as_ref(), false).as_ref());
 
@@ -58,7 +69,7 @@ impl VerbatimUrl {
     }
 
     /// Parse a URL from an absolute path.
-    pub fn from_absolute_path(path: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
+    pub fn parse_absolute_path(path: impl AsRef<str>) -> Result<Self, VerbatimUrlError> {
         // Expand any environment variables.
         let path = PathBuf::from(expand_env_vars(path.as_ref(), false).as_ref());
 
@@ -115,7 +126,9 @@ impl std::str::FromStr for VerbatimUrl {
     type Err = VerbatimUrlError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Self::parse(s).map(|url| url.with_given(s.to_owned()))
+        Self::parse(s)
+            .map(|url| url.with_given(s.to_owned()))
+            .map_err(|e| VerbatimUrlError::Url(s.to_owned(), e))
     }
 }
 
@@ -138,7 +151,7 @@ impl Deref for VerbatimUrl {
 pub enum VerbatimUrlError {
     /// Failed to parse a URL.
     #[error("{0}")]
-    Url(String, #[source] url::ParseError),
+    Url(String, #[source] ParseError),
 
     /// Received a relative path, but no working directory was provided.
     #[error("relative path without a working directory: {0}")]

crates/requirements-txt/src/lib.rs

@@ -70,9 +70,9 @@ enum RequirementsTxtStatement {
     /// `-e`
     EditableRequirement(EditableRequirement),
     /// `--index-url`
-    IndexUrl(Url),
+    IndexUrl(VerbatimUrl),
     /// `--extra-index-url`
-    ExtraIndexUrl(Url),
+    ExtraIndexUrl(VerbatimUrl),
     /// `--find-links`
     FindLinks(FindLink),
     /// `--no-index`
@@ -215,7 +215,7 @@ impl EditableRequirement {
                     // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
                     let path = normalize_url_path(path);
 
-                    VerbatimUrl::from_path(path, working_dir.as_ref())
+                    VerbatimUrl::parse_path(path, working_dir.as_ref())
                 }
 
                 // Ex) `https://download.pytorch.org/whl/torch_stable.html`
@@ -226,11 +226,11 @@ impl EditableRequirement {
                 }
 
                 // Ex) `C:/Users/ferris/wheel-0.42.0.tar.gz`
-                _ => VerbatimUrl::from_path(requirement, working_dir.as_ref()),
+                _ => VerbatimUrl::parse_path(requirement, working_dir.as_ref()),
             }
         } else {
             // Ex) `../editable/`
-            VerbatimUrl::from_path(requirement, working_dir.as_ref())
+            VerbatimUrl::parse_path(requirement, working_dir.as_ref())
         };
 
         // Create a `PathBuf`.
@@ -308,9 +308,9 @@ pub struct RequirementsTxt {
     /// Editables with `-e`.
     pub editables: Vec<EditableRequirement>,
     /// The index URL, specified with `--index-url`.
-    pub index_url: Option<Url>,
+    pub index_url: Option<VerbatimUrl>,
     /// The extra index URLs, specified with `--extra-index-url`.
-    pub extra_index_urls: Vec<Url>,
+    pub extra_index_urls: Vec<VerbatimUrl>,
     /// The find links locations, specified with `--find-links`.
     pub find_links: Vec<FindLink>,
     /// Whether to ignore the index, specified with `--no-index`.
@@ -482,22 +482,26 @@ fn parse_entry(
             .map_err(|err| err.with_offset(start))?;
         RequirementsTxtStatement::EditableRequirement(editable_requirement)
     } else if s.eat_if("-i") || s.eat_if("--index-url") {
-        let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
-        let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
-            source: err,
-            url: url.to_string(),
-            start,
-            end: s.cursor(),
-        })?;
+        let given = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
+        let url = VerbatimUrl::parse(given)
+            .map(|url| url.with_given(given.to_owned()))
+            .map_err(|err| RequirementsTxtParserError::Url {
+                source: err,
+                url: given.to_string(),
+                start,
+                end: s.cursor(),
+            })?;
         RequirementsTxtStatement::IndexUrl(url)
     } else if s.eat_if("--extra-index-url") {
-        let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
-        let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
-            source: err,
-            url: url.to_string(),
-            start,
-            end: s.cursor(),
-        })?;
+        let given = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
+        let url = VerbatimUrl::parse(given)
+            .map(|url| url.with_given(given.to_owned()))
+            .map_err(|err| RequirementsTxtParserError::Url {
+                source: err,
+                url: given.to_string(),
+                start,
+                end: s.cursor(),
+            })?;
         RequirementsTxtStatement::ExtraIndexUrl(url)
     } else if s.eat_if("--no-index") {
         RequirementsTxtStatement::NoIndex

crates/uv-cache/src/wheel.rs

@@ -30,7 +30,7 @@ pub enum WheelCache<'a> {
 impl<'a> WheelCache<'a> {
     fn bucket(&self) -> PathBuf {
         match self {
-            WheelCache::Index(IndexUrl::Pypi) => WheelCacheKind::Pypi.root(),
+            WheelCache::Index(IndexUrl::Pypi(_)) => WheelCacheKind::Pypi.root(),
             WheelCache::Index(url) => WheelCacheKind::Index
                 .root()
                 .join(digest(&CanonicalUrl::new(url))),

crates/uv-client/src/flat_index.rs

@@ -14,6 +14,7 @@ use distribution_types::{
     RegistryBuiltDist, RegistrySourceDist, SourceDist,
 };
 use pep440_rs::Version;
+use pep508_rs::VerbatimUrl;
 use platform_tags::Tags;
 use pypi_types::{Hashes, Yanked};
 use uv_auth::safe_copy_url_auth;
@@ -191,13 +192,14 @@ impl<'a> FlatIndexClient<'a> {
             .await;
         match response {
             Ok(files) => {
+                let index_url = IndexUrl::Url(VerbatimUrl::from_url(url.clone()));
                 let files = files
                     .into_iter()
                     .filter_map(|file| {
                         Some((
                             DistFilename::try_from_normalized_filename(&file.filename)?,
                             file,
-                            IndexUrl::Url(url.clone()),
+                            index_url.clone(),
                         ))
                     })
                     .collect();
@@ -214,6 +216,7 @@ impl<'a> FlatIndexClient<'a> {
     fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
         // Absolute paths are required for the URL conversion.
         let path = fs_err::canonicalize(path)?;
+        let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path));
 
         let mut dists = Vec::new();
         for entry in fs_err::read_dir(path)? {
@@ -249,7 +252,7 @@ impl<'a> FlatIndexClient<'a> {
                 );
                 continue;
             };
-            dists.push((filename, file, IndexUrl::Pypi));
+            dists.push((filename, file, index_url.clone()));
         }
         Ok(FlatIndexEntries::from_entries(dists))
     }

crates/uv-client/src/registry_client.rs

@@ -237,7 +237,7 @@ impl RegistryClient {
         let cache_entry = self.cache.entry(
             CacheBucket::Simple,
             Path::new(&match index {
-                IndexUrl::Pypi => "pypi".to_string(),
+                IndexUrl::Pypi(_) => "pypi".to_string(),
                 IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
             }),
             format!("{package_name}.rkyv"),

crates/uv-dev/src/resolve_cli.rs

@@ -43,8 +43,8 @@ pub(crate) struct ResolveCliArgs {
     cache_args: CacheArgs,
     #[arg(long)]
     exclude_newer: Option<DateTime<Utc>>,
-    #[clap(long, short, default_value = IndexUrl::Pypi.as_str(), env = "UV_INDEX_URL")]
-    index_url: IndexUrl,
+    #[clap(long, short, env = "UV_INDEX_URL")]
+    index_url: Option<IndexUrl>,
     #[clap(long, env = "UV_EXTRA_INDEX_URL")]
     extra_index_url: Vec<IndexUrl>,
     #[clap(long)]
@@ -56,12 +56,8 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
 
     let platform = Platform::current()?;
     let venv = PythonEnvironment::from_virtualenv(platform, &cache)?;
-    let index_locations = IndexLocations::new(
-        Some(args.index_url),
-        args.extra_index_url,
-        args.find_links,
-        false,
-    );
+    let index_locations =
+        IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false);
     let client = RegistryClientBuilder::new(cache.clone())
         .index_urls(index_locations.index_urls())
         .build();

crates/uv-distribution/src/index/registry_wheel_index.rs

@@ -6,6 +6,7 @@ use rustc_hash::FxHashMap;
 
 use distribution_types::{CachedRegistryDist, FlatIndexLocation, IndexLocations, IndexUrl};
 use pep440_rs::Version;
+use pep508_rs::VerbatimUrl;
 use platform_tags::Tags;
 use uv_cache::{Cache, CacheBucket, WheelCache};
 use uv_fs::{directories, symlinks};
@@ -83,7 +84,9 @@ impl<'a> RegistryWheelIndex<'a> {
             .flat_index()
             .filter_map(|flat_index| match flat_index {
                 FlatIndexLocation::Path(_) => None,
-                FlatIndexLocation::Url(url) => Some(IndexUrl::Url(url.clone())),
+                FlatIndexLocation::Url(url) => {
+                    Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
+                }
             })
             .collect();
 

crates/uv/src/commands/pip_compile.rs

@@ -15,7 +15,7 @@ use rustc_hash::FxHashSet;
 use tempfile::tempdir_in;
 use tracing::debug;
 
-use distribution_types::{IndexLocations, LocalEditable};
+use distribution_types::{IndexLocations, LocalEditable, Verbatim};
 use pep508_rs::Requirement;
 use platform_host::Platform;
 use platform_tags::Tags;
@@ -357,11 +357,11 @@ pub(crate) async fn pip_compile(
     // If necessary, include the `--index-url` and `--extra-index-url` locations.
     if include_index_url {
         if let Some(index) = index_locations.index() {
-            writeln!(writer, "--index-url {index}")?;
+            writeln!(writer, "--index-url {}", index.verbatim())?;
             wrote_index = true;
         }
         for extra_index in index_locations.extra_index() {
-            writeln!(writer, "--extra-index-url {extra_index}")?;
+            writeln!(writer, "--extra-index-url {}", extra_index.verbatim())?;
             wrote_index = true;
         }
     }

crates/uv/tests/common/mod.rs

@@ -96,6 +96,13 @@ impl TestContext {
             .arg(EXCLUDE_NEWER)
             .env("VIRTUAL_ENV", self.venv.as_os_str())
             .current_dir(self.temp_dir.path());
+
+        if cfg!(all(windows, debug_assertions)) {
+            // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
+            // default windows stack of 1MB
+            cmd.env("UV_STACK_SIZE", (8 * 1024 * 1024).to_string());
+        }
+
         cmd
     }
 

crates/uv/tests/pip_compile.rs

@@ -2412,7 +2412,14 @@ fn recursive_extras_direct_url() -> Result<()> {
         .chain(INSTA_FILTERS.to_vec())
         .collect();
 
-    uv_snapshot!(filters, Command::new(get_bin())
+    let mut command = Command::new(get_bin());
+    if cfg!(all(windows, debug_assertions)) {
+        // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
+        // default windows stack of 1MB
+        command.env("UV_STACK_SIZE", (2 * 1024 * 1024).to_string());
+    }
+
+    uv_snapshot!(filters, command
             .arg("pip")
             .arg("compile")
             .arg(requirements_in.path())

crates/uv/tests/pip_install.rs

@@ -1035,7 +1035,7 @@ fn install_git_private_https_pat_not_authorized() {
       Caused by: process didn't exit successfully: `git fetch --force --update-head-ok 'https://git:***@github.com/astral-test/uv-private-pypackage' '+HEAD:refs/remotes/origin/HEAD'` (exit status: 128)
     --- stderr
     remote: Support for password authentication was removed on August 13, 2021.
-    remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
+    remote: Please see https://docs.github.com/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
     fatal: Authentication failed for 'https://github.com/astral-test/uv-private-pypackage/'
 
     "###);

crates/uv/tests/pip_sync.rs

@@ -42,6 +42,13 @@ fn command(context: &TestContext) -> Command {
         .arg(context.cache_dir.path())
         .env("VIRTUAL_ENV", context.venv.as_os_str())
         .current_dir(&context.temp_dir);
+
+    if cfg!(all(windows, debug_assertions)) {
+        // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
+        // default windows stack of 1MB
+        command.env("UV_STACK_SIZE", (8 * 1024 * 1024).to_string());
+    }
+
     command
 }
 
@@ -55,6 +62,13 @@ fn uninstall_command(context: &TestContext) -> Command {
         .arg(context.cache_dir.path())
         .env("VIRTUAL_ENV", context.venv.as_os_str())
         .current_dir(&context.temp_dir);
+
+    if cfg!(all(windows, debug_assertions)) {
+        // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the
+        // default windows stack of 1MB
+        command.env("UV_STACK_SIZE", (8 * 1024 * 1024).to_string());
+    }
+
     command
 }