Preserve verbatim URLs (#639)

## Summary

This PR adds a `VerbatimUrl` struct to preserve verbatim URLs throughout
the resolution and installation pipeline. In short, alongside the parsed
`Url`, we also keep the URL as written by the user. This enables us to
display the URL exactly as written by the user, rather than the
serialized path that we use internally.

This will be especially useful once we start expanding environment
variables since, at that point, we'll be able to write the version of
the URL that includes the _unexpected_ environment variable to the
output file.

Cargo.lock

@@ -850,6 +850,7 @@ dependencies = [
  "distribution-filename",
  "fs-err",
  "pep440_rs 0.3.12",
+ "pep508_rs",
  "puffin-cache",
  "puffin-git",
  "puffin-normalize",
@@ -1998,6 +1999,7 @@ dependencies = [
 name = "pep508_rs"
 version = "0.2.3"
 dependencies = [
+ "derivative",
  "indoc",
  "log",
  "once_cell",
@@ -2360,6 +2362,7 @@ dependencies = [
  "http-cache-semantics",
  "install-wheel-rs",
  "pep440_rs 0.3.12",
+ "pep508_rs",
  "puffin-cache",
  "puffin-fs",
  "puffin-normalize",
@@ -2457,6 +2460,7 @@ dependencies = [
  "futures",
  "install-wheel-rs",
  "pep440_rs 0.3.12",
+ "pep508_rs",
  "platform-tags",
  "puffin-cache",
  "puffin-client",

crates/distribution-types/Cargo.toml

@@ -15,6 +15,7 @@ workspace = true
 [dependencies]
 distribution-filename = { path = "../distribution-filename" }
 pep440_rs = { path = "../pep440-rs" }
+pep508_rs = { path = "../pep508-rs" }
 puffin-cache = { path = "../puffin-cache" }
 puffin-git = { path = "../puffin-git" }
 puffin-normalize = { path = "../puffin-normalize" }

crates/distribution-types/src/cached.rs

@@ -1,9 +1,9 @@
 use std::path::{Path, PathBuf};
 
 use anyhow::Result;
-use url::Url;
 
 use distribution_filename::WheelFilename;
+use pep508_rs::VerbatimUrl;
 use puffin_normalize::PackageName;
 
 use crate::direct_url::DirectUrl;
@@ -28,7 +28,7 @@ pub struct CachedRegistryDist {
 #[derive(Debug, Clone)]
 pub struct CachedDirectUrlDist {
     pub filename: WheelFilename,
-    pub url: Url,
+    pub url: VerbatimUrl,
     pub path: PathBuf,
 }
 
@@ -118,14 +118,14 @@ impl CachedDist {
     pub fn direct_url(&self) -> Result<Option<DirectUrl>> {
         match self {
             CachedDist::Registry(_) => Ok(None),
-            CachedDist::Url(dist) => DirectUrl::try_from(&dist.url).map(Some),
+            CachedDist::Url(dist) => DirectUrl::try_from(dist.url.raw()).map(Some),
         }
     }
 }
 
 impl CachedDirectUrlDist {
     /// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`Url`], and [`Path`].
-    pub fn from_url(filename: WheelFilename, url: Url, path: PathBuf) -> Self {
+    pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self {
         Self {
             filename,
             url,
@@ -172,7 +172,7 @@ impl CachedWheel {
     }
 
     /// Convert a [`CachedWheel`] into a [`CachedDirectUrlDist`].
-    pub fn into_url_dist(self, url: Url) -> CachedDirectUrlDist {
+    pub fn into_url_dist(self, url: VerbatimUrl) -> CachedDirectUrlDist {
         CachedDirectUrlDist {
             filename: self.filename,
             url,

crates/distribution-types/src/lib.rs

@@ -45,6 +45,7 @@ use url::Url;
 
 use distribution_filename::WheelFilename;
 use pep440_rs::Version;
+use pep508_rs::VerbatimUrl;
 use puffin_normalize::PackageName;
 use pypi_types::{File, IndexUrl};
 
@@ -68,7 +69,7 @@ pub enum VersionOrUrl<'a> {
     /// A PEP 440 version specifier, used to identify a distribution in a registry.
     Version(&'a Version),
     /// A URL, used to identify a distribution at an arbitrary location.
-    Url(&'a Url),
+    Url(&'a VerbatimUrl),
 }
 
 impl std::fmt::Display for VersionOrUrl<'_> {
@@ -123,14 +124,14 @@ pub struct DirectUrlBuiltDist {
     /// We require that wheel urls end in the full wheel filename, e.g.
     /// `https://example.org/packages/flask-3.0.0-py3-none-any.whl`
     pub filename: WheelFilename,
-    pub url: Url,
+    pub url: VerbatimUrl,
 }
 
 /// A built distribution (wheel) that exists in a local directory.
 #[derive(Debug, Clone)]
 pub struct PathBuiltDist {
     pub filename: WheelFilename,
-    pub url: Url,
+    pub url: VerbatimUrl,
     pub path: PathBuf,
 }
 
@@ -149,21 +150,21 @@ pub struct DirectUrlSourceDist {
     /// Unlike [`DirectUrlBuiltDist`], we can't require a full filename with a version here, people
     /// like using e.g. `foo @ https://github.com/org/repo/archive/master.zip`
     pub name: PackageName,
-    pub url: Url,
+    pub url: VerbatimUrl,
 }
 
 /// A source distribution that exists in a Git repository.
 #[derive(Debug, Clone)]
 pub struct GitSourceDist {
     pub name: PackageName,
-    pub url: Url,
+    pub url: VerbatimUrl,
 }
 
 /// A source distribution that exists in a local directory.
 #[derive(Debug, Clone)]
 pub struct PathSourceDist {
     pub name: PackageName,
-    pub url: Url,
+    pub url: VerbatimUrl,
     pub path: PathBuf,
 }
 
@@ -191,7 +192,7 @@ impl Dist {
     }
 
     /// Create a [`Dist`] for a URL-based distribution.
-    pub fn from_url(name: PackageName, url: Url) -> Result<Self, Error> {
+    pub fn from_url(name: PackageName, url: VerbatimUrl) -> Result<Self, Error> {
         if url.scheme().starts_with("git+") {
             return Ok(Self::Source(SourceDist::Git(GitSourceDist { name, url })));
         }
@@ -200,9 +201,9 @@ impl Dist {
             // Store the canonicalized path.
             let path = url
                 .to_file_path()
-                .map_err(|()| Error::UrlFilename(url.clone()))?
+                .map_err(|()| Error::UrlFilename(url.to_url()))?
                 .canonicalize()
-                .map_err(|err| Error::NotFound(url.clone(), err))?;
+                .map_err(|err| Error::NotFound(url.to_url(), err))?;
             return if path
                 .extension()
                 .is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
@@ -268,11 +269,18 @@ impl SourceDist {
     #[must_use]
     pub fn with_url(self, url: Url) -> Self {
         match self {
-            SourceDist::DirectUrl(dist) => {
-                SourceDist::DirectUrl(DirectUrlSourceDist { url, ..dist })
-            }
-            SourceDist::Git(dist) => SourceDist::Git(GitSourceDist { url, ..dist }),
-            SourceDist::Path(dist) => SourceDist::Path(PathSourceDist { url, ..dist }),
+            SourceDist::DirectUrl(dist) => SourceDist::DirectUrl(DirectUrlSourceDist {
+                url: VerbatimUrl::unknown(url),
+                ..dist
+            }),
+            SourceDist::Git(dist) => SourceDist::Git(GitSourceDist {
+                url: VerbatimUrl::unknown(url),
+                ..dist
+            }),
+            SourceDist::Path(dist) => SourceDist::Path(PathSourceDist {
+                url: VerbatimUrl::unknown(url),
+                ..dist
+            }),
             dist @ SourceDist::Registry(_) => dist,
         }
     }

crates/pep508-rs/Cargo.toml

@@ -20,6 +20,7 @@ crate-type = ["cdylib", "rlib"]
 pep440_rs = { path = "../pep440-rs" }
 puffin-normalize = { path = "../puffin-normalize" }
 
+derivative = { workspace = true }
 once_cell = { workspace = true }
 pyo3 = { workspace = true, optional = true, features = ["abi3", "extension-module"] }
 pyo3-log = { workspace = true, optional = true }

crates/pep508-rs/src/lib.rs

@@ -35,7 +35,6 @@ use pyo3::{
 use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
 use thiserror::Error;
 use unicode_width::UnicodeWidthStr;
-use url::Url;
 
 pub use marker::{
     MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
@@ -43,8 +42,10 @@ pub use marker::{
 };
 use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
 use puffin_normalize::{ExtraName, PackageName};
+pub use verbatim_url::VerbatimUrl;
 
 mod marker;
+mod verbatim_url;
 
 /// Error with a span attached. Not that those aren't `String` but `Vec<char>` indices.
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -62,12 +63,12 @@ pub struct Pep508Error {
 /// Either we have an error string from our parser or an upstream error from `url`
 #[derive(Debug, Error, Clone, Eq, PartialEq)]
 pub enum Pep508ErrorSource {
-    /// An error from our parser
+    /// An error from our parser.
     #[error("{0}")]
     String(String),
-    /// A url parsing error
+    /// A URL parsing error.
     #[error(transparent)]
-    UrlError(#[from] url::ParseError),
+    UrlError(#[from] verbatim_url::Error),
 }
 
 impl Display for Pep508Error {
@@ -397,7 +398,7 @@ pub enum VersionOrUrl {
     /// A PEP 440 version specifier set
     VersionSpecifier(VersionSpecifiers),
     /// A installable URL
-    Url(Url),
+    Url(VerbatimUrl),
 }
 
 /// A `Vec<char>` and an index inside of it. Like [String], but with utf-8 aware indexing
@@ -678,7 +679,7 @@ fn parse_url(chars: &mut CharIter) -> Result<VersionOrUrl, Pep508Error> {
             input: chars.copy_chars(),
         });
     }
-    let url = Url::parse(&url).map_err(|err| Pep508Error {
+    let url = VerbatimUrl::parse(url).map_err(|err| Pep508Error {
         message: Pep508ErrorSource::UrlError(err),
         start,
         len,
@@ -888,7 +889,6 @@ mod tests {
     use std::str::FromStr;
 
     use indoc::indoc;
-    use url::Url;
 
     use pep440_rs::{Operator, Version, VersionSpecifier};
     use puffin_normalize::{ExtraName, PackageName};
@@ -897,7 +897,7 @@ mod tests {
         parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
         MarkerValueString, MarkerValueVersion,
     };
-    use crate::{CharIter, Requirement, VersionOrUrl};
+    use crate::{CharIter, Requirement, VerbatimUrl, VersionOrUrl};
 
     fn assert_err(input: &str, error: &str) {
         assert_eq!(Requirement::from_str(input).unwrap_err().to_string(), error);
@@ -1151,7 +1151,7 @@ mod tests {
             name: PackageName::from_str("pip").unwrap(),
             extras: None,
             marker: None,
-            version_or_url: Some(VersionOrUrl::Url(Url::parse(url).unwrap())),
+            version_or_url: Some(VersionOrUrl::Url(VerbatimUrl::from_str(url).unwrap())),
         };
         assert_eq!(pip_url, expected);
     }

crates/pep508-rs/src/verbatim_url.rs

@@ -0,0 +1,75 @@
+use std::ops::Deref;
+use url::Url;
+
+/// A wrapper around [`Url`] that preserves the original string.
+#[derive(Debug, Clone, Eq, derivative::Derivative)]
+#[derivative(PartialEq, Hash)]
+pub struct VerbatimUrl {
+    /// The parsed URL.
+    url: Url,
+    /// The URL as it was provided by the user.
+    #[derivative(PartialEq = "ignore")]
+    #[derivative(Hash = "ignore")]
+    given: Option<String>,
+}
+
+impl VerbatimUrl {
+    /// Parse a URL from a string.
+    pub fn parse(given: String) -> Result<Self, Error> {
+        let url = Url::parse(&given)?;
+        Ok(Self {
+            given: Some(given),
+            url,
+        })
+    }
+
+    /// Return the underlying [`Url`].
+    pub fn raw(&self) -> &Url {
+        &self.url
+    }
+
+    /// Convert a [`VerbatimUrl`] into a [`Url`].
+    pub fn to_url(&self) -> Url {
+        self.url.clone()
+    }
+
+    /// Create a [`VerbatimUrl`] from a [`Url`].
+    ///
+    /// This method should be used sparingly (ideally, not at all), as it represents a loss of the
+    /// verbatim representation.
+    pub fn unknown(url: Url) -> Self {
+        Self { given: None, url }
+    }
+}
+
+impl std::str::FromStr for VerbatimUrl {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::parse(s.to_owned())
+    }
+}
+
+impl std::fmt::Display for VerbatimUrl {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if let Some(given) = &self.given {
+            given.fmt(f)
+        } else {
+            self.url.fmt(f)
+        }
+    }
+}
+
+impl Deref for VerbatimUrl {
+    type Target = Url;
+
+    fn deref(&self) -> &Self::Target {
+        &self.url
+    }
+}
+
+#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)]
+pub enum Error {
+    #[error(transparent)]
+    Url(#[from] url::ParseError),
+}

crates/puffin-cli/tests/pip_compile.rs

@@ -2438,3 +2438,55 @@ fn missing_url_extra() -> Result<()> {
 
     Ok(())
 }
+
+/// Resolve a dependency from a URL, preserving the exact casing of the URL as specified in the
+/// requirements file.
+#[test]
+fn preserve_url() -> Result<()> {
+    let temp_dir = TempDir::new()?;
+    let cache_dir = TempDir::new()?;
+    let venv = create_venv_py312(&temp_dir, &cache_dir);
+
+    let requirements_in = temp_dir.child("requirements.in");
+    requirements_in.write_str("flask @ https://files.PYTHONHOSTED.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl")?;
+
+    insta::with_settings!({
+        filters => INSTA_FILTERS.to_vec()
+    }, {
+        assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+            .arg("pip-compile")
+            .arg("requirements.in")
+            .arg("--cache-dir")
+            .arg(cache_dir.path())
+            .arg("--exclude-newer")
+            .arg(EXCLUDE_NEWER)
+            .env("VIRTUAL_ENV", venv.as_os_str())
+            .current_dir(&temp_dir), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+        # This file was autogenerated by Puffin v0.0.1 via the following command:
+        #    puffin pip-compile requirements.in --cache-dir [CACHE_DIR]
+        blinker==1.7.0
+            # via flask
+        click==8.1.7
+            # via flask
+        flask @ https://files.PYTHONHOSTED.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
+        itsdangerous==2.1.2
+            # via flask
+        jinja2==3.1.2
+            # via flask
+        markupsafe==2.1.3
+            # via
+            #   jinja2
+            #   werkzeug
+        werkzeug==3.0.1
+            # via flask
+
+        ----- stderr -----
+        Resolved 7 packages in [TIME]
+        "###);
+    });
+
+    Ok(())
+}

crates/puffin-client/Cargo.toml

@@ -33,5 +33,7 @@ tracing = { workspace = true }
 url = { workspace = true }
 
 [dev-dependencies]
+pep508_rs = { path = "../pep508-rs" }
+
 anyhow = { workspace = true }
 tokio = { workspace = true, features = ["fs", "macros"] }

crates/puffin-client/tests/remote_metadata.rs

@@ -1,10 +1,10 @@
 use std::str::FromStr;
 
 use anyhow::Result;
-use url::Url;
 
 use distribution_filename::WheelFilename;
 use distribution_types::{BuiltDist, DirectUrlBuiltDist};
+use pep508_rs::VerbatimUrl;
 use puffin_cache::Cache;
 use puffin_client::RegistryClientBuilder;
 
@@ -20,7 +20,7 @@ async fn remote_metadata_with_and_without_cache() -> Result<()> {
         let filename = WheelFilename::from_str(url.rsplit_once('/').unwrap().1)?;
         let dist = BuiltDist::DirectUrl(DirectUrlBuiltDist {
             filename,
-            url: Url::parse(url).unwrap(),
+            url: VerbatimUrl::from_str(url).unwrap(),
         });
         let metadata = client.wheel_metadata(&dist).await.unwrap();
         assert_eq!(metadata.version.to_string(), "4.66.1");

crates/puffin-dev/src/wheel_metadata.rs

@@ -2,16 +2,16 @@ use std::str::FromStr;
 
 use anyhow::Result;
 use clap::Parser;
-use url::Url;
 
 use distribution_filename::WheelFilename;
 use distribution_types::{BuiltDist, DirectUrlBuiltDist};
+use pep508_rs::VerbatimUrl;
 use puffin_cache::{Cache, CacheArgs};
 use puffin_client::RegistryClientBuilder;
 
 #[derive(Parser)]
 pub(crate) struct WheelMetadataArgs {
-    url: Url,
+    url: VerbatimUrl,
     #[command(flatten)]
     cache_args: CacheArgs,
 }

crates/puffin-distribution/Cargo.toml

@@ -17,6 +17,7 @@ distribution-filename = { path = "../distribution-filename", features = ["serde"
 distribution-types = { path = "../distribution-types" }
 install-wheel-rs = { path = "../install-wheel-rs" }
 pep440_rs = { path = "../pep440-rs" }
+pep508_rs = { path = "../pep508-rs" }
 platform-tags = { path = "../platform-tags" }
 puffin-cache = { path = "../puffin-cache" }
 puffin-client = { path = "../puffin-client" }

crates/puffin-distribution/src/distribution_database.rs

@@ -261,6 +261,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
 
                 // Insert the `precise` URL, if it exists.
                 let precise = self.precise(source_dist).await?;
+
                 let source_dist = match precise.as_ref() {
                     Some(url) => Cow::Owned(source_dist.clone().with_url(url.clone())),
                     None => Cow::Borrowed(source_dist),
@@ -280,17 +281,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
     /// This method takes into account various normalizations that are independent from the Git
     /// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+`
     /// prefix kinds.
-    pub async fn precise(
-        &self,
-        dist: &SourceDist,
-    ) -> Result<Option<Url>, DistributionDatabaseError> {
+    async fn precise(&self, dist: &SourceDist) -> Result<Option<Url>, DistributionDatabaseError> {
         let SourceDist::Git(source_dist) = dist else {
             return Ok(None);
         };
         let git_dir = self.build_context.cache().bucket(CacheBucket::Git);
 
-        let DirectGitUrl { url, subdirectory } =
-            DirectGitUrl::try_from(&source_dist.url).map_err(DistributionDatabaseError::Git)?;
+        let DirectGitUrl { url, subdirectory } = DirectGitUrl::try_from(source_dist.url.raw())
+            .map_err(DistributionDatabaseError::Git)?;
 
         // If the commit already contains a complete SHA, short-circuit.
         if url.precise().is_some() {
@@ -310,6 +308,6 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
         let url = precise.into_git();
 
         // Re-encode as a URL.
-        Ok(Some(DirectGitUrl { url, subdirectory }.into()))
+        Ok(Some(Url::from(DirectGitUrl { url, subdirectory })))
     }
 }

crates/puffin-distribution/src/source_dist.rs

@@ -161,7 +161,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
                     .filename()
                     .unwrap_or(direct_url_source_dist.url.path());
                 let DirectArchiveUrl { url, subdirectory } =
-                    DirectArchiveUrl::from(&direct_url_source_dist.url);
+                    DirectArchiveUrl::from(direct_url_source_dist.url.raw());
 
                 // For direct URLs, cache directly under the hash of the URL itself.
                 let cache_shard = self.build_context.cache().shard(

crates/puffin-installer/src/plan.rs

@@ -104,7 +104,7 @@ impl InstallPlan {
                         // If the requirement comes from a direct URL, check by URL.
                         Some(VersionOrUrl::Url(url)) => {
                             if let InstalledDist::Url(distribution) = &distribution {
-                                if let Ok(direct_url) = DirectUrl::try_from(url) {
+                                if let Ok(direct_url) = DirectUrl::try_from(url.raw()) {
                                     if let Ok(direct_url) =
                                         pypi_types::DirectUrl::try_from(&direct_url)
                                     {

crates/puffin-resolver/src/pubgrub/distribution.rs

@@ -1,6 +1,5 @@
-use url::Url;
-
 use distribution_types::{Metadata, VersionOrUrl};
+use pep508_rs::VerbatimUrl;
 use puffin_normalize::PackageName;
 
 use crate::pubgrub::PubGrubVersion;
@@ -8,7 +7,7 @@ use crate::pubgrub::PubGrubVersion;
 #[derive(Debug)]
 pub(crate) enum PubGrubDistribution<'a> {
     Registry(&'a PackageName, &'a PubGrubVersion),
-    Url(&'a PackageName, &'a Url),
+    Url(&'a PackageName, &'a VerbatimUrl),
 }
 
 impl<'a> PubGrubDistribution<'a> {
@@ -16,7 +15,7 @@ impl<'a> PubGrubDistribution<'a> {
         Self::Registry(name, version)
     }
 
-    pub(crate) fn from_url(name: &'a PackageName, url: &'a Url) -> Self {
+    pub(crate) fn from_url(name: &'a PackageName, url: &'a VerbatimUrl) -> Self {
         Self::Url(name, url)
     }
 }

crates/puffin-resolver/src/pubgrub/package.rs

@@ -1,6 +1,6 @@
 use derivative::Derivative;
-use url::Url;
 
+use pep508_rs::VerbatimUrl;
 use puffin_normalize::{ExtraName, PackageName};
 
 /// A PubGrub-compatible wrapper around a "Python package", with two notable characteristics:
@@ -66,7 +66,7 @@ pub enum PubGrubPackage {
         #[derivative(PartialEq = "ignore")]
         #[derivative(PartialOrd = "ignore")]
         #[derivative(Hash = "ignore")]
-        Option<Url>,
+        Option<VerbatimUrl>,
     ),
 }
 

crates/puffin-resolver/src/resolution.rs

@@ -13,7 +13,7 @@ use url::Url;
 
 use distribution_types::{BuiltDist, Dist, Metadata, PackageId, SourceDist};
 use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
-use pep508_rs::{Requirement, VersionOrUrl};
+use pep508_rs::{Requirement, VerbatimUrl, VersionOrUrl};
 use puffin_normalize::{ExtraName, PackageName};
 use puffin_traits::OnceMap;
 use pypi_types::Metadata21;
@@ -109,9 +109,10 @@ impl ResolutionGraph {
                     inverse.insert(package_name, index);
                 }
                 PubGrubPackage::Package(package_name, None, Some(url)) => {
-                    let url = redirects
-                        .get(url)
-                        .map_or_else(|| url.clone(), |url| url.value().clone());
+                    let url = redirects.get(url).map_or_else(
+                        || url.clone(),
+                        |url| VerbatimUrl::unknown(url.value().clone()),
+                    );
                     let pinned_package = Dist::from_url(package_name.clone(), url)?;
 
                     let index = petgraph.add_node(pinned_package);
@@ -149,9 +150,10 @@ impl ResolutionGraph {
                     let metadata = entry.value();
 
                     if !metadata.provides_extras.contains(extra) {
-                        let url = redirects
-                            .get(url)
-                            .map_or_else(|| url.clone(), |url| url.value().clone());
+                        let url = redirects.get(url).map_or_else(
+                            || url.clone(),
+                            |url| VerbatimUrl::unknown(url.value().clone()),
+                        );
                         let pinned_package = Dist::from_url(package_name.clone(), url)?;
 
                         diagnostics.push(Diagnostic::MissingExtra {

crates/puffin-resolver/src/resolver.rs

@@ -19,7 +19,7 @@ use url::Url;
 
 use distribution_filename::WheelFilename;
 use distribution_types::{BuiltDist, Dist, Metadata, PackageId, SourceDist, VersionOrUrl};
-use pep508_rs::{MarkerEnvironment, Requirement};
+use pep508_rs::{MarkerEnvironment, Requirement, VerbatimUrl};
 use platform_tags::Tags;
 use puffin_cache::CanonicalUrl;
 use puffin_client::RegistryClient;
@@ -208,6 +208,7 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
                         None
                     }
                 })
+                .map(VerbatimUrl::raw)
                 .collect(),
             project: manifest.project,
             requirements: manifest.requirements,
@@ -488,11 +489,11 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
                 if !self.allowed_urls.contains(url) {
                     return Err(ResolveError::DisallowedUrl(
                         package_name.clone(),
-                        url.clone(),
+                        url.to_url(),
                     ));
                 }
 
-                if let Ok(wheel_filename) = WheelFilename::try_from(url) {
+                if let Ok(wheel_filename) = WheelFilename::try_from(url.raw()) {
                     // If the URL is that of a wheel, extract the version.
                     let version = PubGrubVersion::from(wheel_filename.version);
                     if range.contains(&version) {
@@ -669,13 +670,13 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> {
                     if let Some(precise) = precise {
                         match distribution {
                             SourceDist::DirectUrl(sdist) => {
-                                self.index.redirects.done(sdist.url.clone(), precise);
+                                self.index.redirects.done(sdist.url.to_url(), precise);
                             }
                             SourceDist::Git(sdist) => {
-                                self.index.redirects.done(sdist.url.clone(), precise);
+                                self.index.redirects.done(sdist.url.to_url(), precise);
                             }
                             SourceDist::Path(sdist) => {
-                                self.index.redirects.done(sdist.url.clone(), precise);
+                                self.index.redirects.done(sdist.url.to_url(), precise);
                             }
                             SourceDist::Registry(_) => {}
                         }

crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap

@@ -30,20 +30,25 @@ RequirementsTxt {
                 ),
                 version_or_url: Some(
                     Url(
-                        Url {
-                            scheme: "https",
-                            cannot_be_a_base: false,
-                            username: "",
-                            password: None,
-                            host: Some(
-                                Domain(
-                                    "github.com",
+                        VerbatimUrl {
+                            url: Url {
+                                scheme: "https",
+                                cannot_be_a_base: false,
+                                username: "",
+                                password: None,
+                                host: Some(
+                                    Domain(
+                                        "github.com",
+                                    ),
                                 ),
+                                port: None,
+                                path: "/pandas-dev/pandas",
+                                query: None,
+                                fragment: None,
+                            },
+                            given: Some(
+                                "https://github.com/pandas-dev/pandas",
                             ),
-                            port: None,
-                            path: "/pandas-dev/pandas",
-                            query: None,
-                            fragment: None,
                         },
                     ),
                 ),

crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap

@@ -30,20 +30,25 @@ RequirementsTxt {
                 ),
                 version_or_url: Some(
                     Url(
-                        Url {
-                            scheme: "https",
-                            cannot_be_a_base: false,
-                            username: "",
-                            password: None,
-                            host: Some(
-                                Domain(
-                                    "github.com",
+                        VerbatimUrl {
+                            url: Url {
+                                scheme: "https",
+                                cannot_be_a_base: false,
+                                username: "",
+                                password: None,
+                                host: Some(
+                                    Domain(
+                                        "github.com",
+                                    ),
                                 ),
+                                port: None,
+                                path: "/pandas-dev/pandas",
+                                query: None,
+                                fragment: None,
+                            },
+                            given: Some(
+                                "https://github.com/pandas-dev/pandas",
                             ),
-                            port: None,
-                            path: "/pandas-dev/pandas",
-                            query: None,
-                            fragment: None,
                         },
                     ),
                 ),

crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap

@@ -30,20 +30,25 @@ RequirementsTxt {
                 ),
                 version_or_url: Some(
                     Url(
-                        Url {
-                            scheme: "https",
-                            cannot_be_a_base: false,
-                            username: "",
-                            password: None,
-                            host: Some(
-                                Domain(
-                                    "github.com",
+                        VerbatimUrl {
+                            url: Url {
+                                scheme: "https",
+                                cannot_be_a_base: false,
+                                username: "",
+                                password: None,
+                                host: Some(
+                                    Domain(
+                                        "github.com",
+                                    ),
                                 ),
+                                port: None,
+                                path: "/pandas-dev/pandas",
+                                query: None,
+                                fragment: None,
+                            },
+                            given: Some(
+                                "https://github.com/pandas-dev/pandas",
                             ),
-                            port: None,
-                            path: "/pandas-dev/pandas",
-                            query: None,
-                            fragment: None,
                         },
                     ),
                 ),