mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-08-04 10:58:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Support parsing --find-links, --index-url, and --extra-index-url in requirements.txt (#1146)

Browse source 
## Summary

This PR adds support for `--find-links`, `--index-url`, and
`--extra-index-url` arguments when specified in a `requirements.txt`.

It's a mostly-straightforward change. The only uncertain piece is what
to do when multiple files include these flags, and/or when we include
them on the CLI and in other files.

In general:

- If _anything_ specifies `--no-index`, we respect it.
- We combine all `--extra-index-url` and `--find-links` across all
sources, since those are just vectors.
- If we see multiple `--index-url` in requirements files, we error.
- We respect the `--index-url` from the command line over any provided
in a requirements file.

(`pip-compile` seems to just pick one semi-arbitrarily when multiple are
provided.)

Closes https://github.com/astral-sh/puffin/issues/1143.
This commit is contained in:
Charlie Marsh 2024-01-29 07:06:40 -08:00 • committed by GitHub
parent 4b9daf9604
commit 67a09649f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 656 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

36
crates/distribution-types/src/index_url.rs
View file

@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
use url::Url;
use pep508_rs::split_scheme;
use puffin_fs::normalize_url_path;
static PYPI_URL: Lazy<Url> = Lazy::new(|| Url::parse("https://pypi.org/simple").unwrap());
@ -89,7 +90,11 @@ impl FromStr for FlatIndexLocation {
if scheme == "file" {
// Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/`
let path = path.strip_prefix("//").unwrap_or(path);
let path = PathBuf::from(path);
// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);
let path = PathBuf::from(path.as_ref());
Ok(Self::Path(path))
} else {
// Ex) `https://download.pytorch.org/whl/torch_stable.html`
@ -162,6 +167,35 @@ impl IndexLocations {
}
}
}
/// Combine a set of index locations.
///
/// If either the current or the other index locations have `no_index` set, the result will
/// have `no_index` set.
///
/// If the current index location has an `index` set, it will be preserved.
#[must_use]
pub fn combine(
self,
index: Option<IndexUrl>,
extra_index: Vec<IndexUrl>,
flat_index: Vec<FlatIndexLocation>,
no_index: bool,
) -> Self {
if no_index {
Self {
index: None,
extra_index: Vec::new(),
flat_index,
}
} else {
Self {
index: self.index.or(index),
extra_index: self.extra_index.into_iter().chain(extra_index).collect(),
flat_index: self.flat_index.into_iter().chain(flat_index).collect(),
}
}
}
}
impl<'a> IndexLocations {

1
crates/pep508-rs/Cargo.toml
View file

@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
pep440_rs = { path = "../pep440-rs" }
puffin-fs = { path = "../puffin-fs" }
puffin-normalize = { path = "../puffin-normalize" }
derivative = { workspace = true }

57
crates/pep508-rs/src/lib.rs
View file

@ -16,7 +16,6 @@
#![deny(missing_docs)]
use std::borrow::Cow;
#[cfg(feature = "pyo3")]
use std::collections::hash_map::DefaultHasher;
use std::collections::HashSet;
@ -43,6 +42,7 @@ pub use marker::{
MarkerValueString, MarkerValueVersion, MarkerWarningKind, StringVersion,
};
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use puffin_fs::normalize_url_path;
#[cfg(feature = "pyo3")]
use puffin_normalize::InvalidNameError;
use puffin_normalize::{ExtraName, PackageName};
@ -745,47 +745,26 @@ fn preprocess_url(
) -> Result<VerbatimUrl, Pep508Error> {
let url = if let Some((scheme, path)) = split_scheme(url) {
if scheme == "file" {
if let Some(path) = path.strip_prefix("//") {
let path = if cfg!(windows) {
// Transform `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`
Cow::Owned(
path.strip_prefix('/')
.unwrap_or(path)
.replace('/', std::path::MAIN_SEPARATOR_STR),
)
} else {
Cow::Borrowed(path)
};
// Ex) `file:///home/ferris/project/scripts/...`
if let Some(working_dir) = working_dir {
VerbatimUrl::from_path(path, working_dir).with_given(url.to_string())
} else {
VerbatimUrl::from_absolute_path(path)
.map_err(|err| Pep508Error {
message: Pep508ErrorSource::UrlError(err),
start,
len,
input: cursor.to_string(),
})?
.with_given(url.to_string())
}
// Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/`.
let path = path.strip_prefix("//").unwrap_or(path);
// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);
if let Some(working_dir) = working_dir {
VerbatimUrl::from_path(path, working_dir).with_given(url.to_string())
} else {
// Ex) `file:../editable/`
if let Some(working_dir) = working_dir {
VerbatimUrl::from_path(path, working_dir).with_given(url.to_string())
} else {
VerbatimUrl::from_absolute_path(path)
.map_err(|err| Pep508Error {
message: Pep508ErrorSource::UrlError(err),
start,
len,
input: cursor.to_string(),
})?
.with_given(url.to_string())
}
VerbatimUrl::from_absolute_path(path)
.map_err(|err| Pep508Error {
message: Pep508ErrorSource::UrlError(err),
start,
len,
input: cursor.to_string(),
})?
.with_given(url.to_string())
}
} else {
// Ex) `https://...`
// Ex) `https://download.pytorch.org/whl/torch_stable.html`
VerbatimUrl::from_str(url).map_err(|err| Pep508Error {
message: Pep508ErrorSource::UrlError(err),
start,

51
crates/puffin-fs/src/path.rs
View file

@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::path::Path;
pub trait NormalizedDisplay {
@ -13,3 +14,53 @@ impl<T: AsRef<Path>> NormalizedDisplay for T {
dunce::simplified(self.as_ref()).display()
}
}
/// Normalize the `path` component of a URL for use as a file path.
///
/// For example, on Windows, transforms `C:\Users\ferris\wheel-0.42.0.tar.gz` to
/// `/C:/Users/ferris/wheel-0.42.0.tar.gz`.
///
/// On other platforms, this is a no-op.
pub fn normalize_url_path(path: &str) -> Cow<'_, str> {
if cfg!(windows) {
Cow::Owned(
path.strip_prefix('/')
.unwrap_or(path)
.replace('/', std::path::MAIN_SEPARATOR_STR),
)
} else {
Cow::Borrowed(path)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize() {
if cfg!(windows) {
assert_eq!(
normalize_url_path("/C:/Users/ferris/wheel-0.42.0.tar.gz"),
"C:\\Users\\ferris\\wheel-0.42.0.tar.gz"
);
} else {
assert_eq!(
normalize_url_path("/C:/Users/ferris/wheel-0.42.0.tar.gz"),
"/C:/Users/ferris/wheel-0.42.0.tar.gz"
);
}
if cfg!(windows) {
assert_eq!(
normalize_url_path("./ferris/wheel-0.42.0.tar.gz"),
".\\ferris\\wheel-0.42.0.tar.gz"
);
} else {
assert_eq!(
normalize_url_path("./ferris/wheel-0.42.0.tar.gz"),
"./ferris/wheel-0.42.0.tar.gz"
);
}
}
}

8
crates/puffin/src/commands/pip_compile.rs
View file

@ -85,9 +85,17 @@ pub(crate) async fn pip_compile(
constraints,
overrides,
editables,
index_url,
extra_index_urls,
no_index,
find_links,
extras: used_extras,
} = RequirementsSpecification::from_sources(requirements, constraints, overrides, &extras)?;
// Incorporate any index locations from the provided sources.
let index_locations =
index_locations.combine(index_url, extra_index_urls, find_links, no_index);
// Check that all provided extras are used
if let ExtrasSpecification::Some(extras) = extras {
let mut unused_extras = extras

8
crates/puffin/src/commands/pip_install.rs
View file

@ -66,9 +66,17 @@ pub(crate) async fn pip_install(
constraints,
overrides,
editables,
index_url,
extra_index_urls,
no_index,
find_links,
extras: used_extras,
} = specification(requirements, constraints, overrides, extras)?;
// Incorporate any index locations from the provided sources.
let index_locations =
index_locations.combine(index_url, extra_index_urls, find_links, no_index);
// Check that all provided extras are used
if let ExtrasSpecification::Some(extras) = extras {
let mut unused_extras = extras

18
crates/puffin/src/commands/pip_sync.rs
View file

@ -44,13 +44,29 @@ pub(crate) async fn pip_sync(
let start = std::time::Instant::now();
// Read all requirements from the provided sources.
let (requirements, editables) = RequirementsSpecification::requirements_and_editables(sources)?;
let RequirementsSpecification {
project: _project,
requirements,
constraints: _constraints,
overrides: _overrides,
editables,
index_url,
extra_index_urls,
no_index,
find_links,
extras: _extras,
} = RequirementsSpecification::from_simple_sources(sources)?;
let num_requirements = requirements.len() + editables.len();
if num_requirements == 0 {
writeln!(printer, "No requirements found")?;
return Ok(ExitStatus::Success);
}
// Incorporate any index locations from the provided sources.
let index_locations =
index_locations.combine(index_url, extra_index_urls, find_links, no_index);
// Detect the current Python interpreter.
let platform = Platform::current()?;
let venv = Virtualenv::from_env(platform, &cache)?;

13
crates/puffin/src/commands/pip_uninstall.rs
View file

@ -23,7 +23,18 @@ pub(crate) async fn pip_uninstall(
let start = std::time::Instant::now();
// Read all requirements from the provided sources.
let (requirements, editables) = RequirementsSpecification::requirements_and_editables(sources)?;
let RequirementsSpecification {
project: _project,
requirements,
constraints: _constraints,
overrides: _overrides,
editables,
index_url: _index_url,
extra_index_urls: _extra_index_urls,
no_index: _no_index,
find_links: _find_links,
extras: _extras,
} = RequirementsSpecification::from_simple_sources(sources)?;
// Detect the current Python interpreter.
let platform = Platform::current()?;

83
crates/puffin/src/requirements.rs
View file

@ -7,10 +7,11 @@ use anyhow::{Context, Result};
use fs_err as fs;
use rustc_hash::FxHashSet;
use distribution_types::{FlatIndexLocation, IndexUrl};
use pep508_rs::Requirement;
use puffin_fs::NormalizedDisplay;
use puffin_normalize::{ExtraName, PackageName};
use requirements_txt::{EditableRequirement, RequirementsTxt};
use requirements_txt::{EditableRequirement, FindLink, RequirementsTxt};
#[derive(Debug)]
pub(crate) enum RequirementsSource {
@ -67,6 +68,14 @@ pub(crate) struct RequirementsSpecification {
pub(crate) editables: Vec<EditableRequirement>,
/// The extras used to collect requirements.
pub(crate) extras: FxHashSet<ExtraName>,
/// The index URL to use for fetching packages.
pub(crate) index_url: Option<IndexUrl>,
/// The extra index URLs to use for fetching packages.
pub(crate) extra_index_urls: Vec<IndexUrl>,
/// Whether to disallow index usage.
pub(crate) no_index: bool,
/// The `--find-links` locations to use for fetching packages.
pub(crate) find_links: Vec<FlatIndexLocation>,
}
impl RequirementsSpecification {
@ -86,6 +95,10 @@ impl RequirementsSpecification {
overrides: vec![],
editables: vec![],
extras: FxHashSet::default(),
index_url: None,
extra_index_urls: vec![],
no_index: false,
find_links: vec![],
}
}
RequirementsSource::Editable(name) => {
@ -98,6 +111,10 @@ impl RequirementsSpecification {
overrides: vec![],
editables: vec![requirement],
extras: FxHashSet::default(),
index_url: None,
extra_index_urls: vec![],
no_index: false,
find_links: vec![],
}
}
RequirementsSource::RequirementsTxt(path) => {
@ -113,6 +130,21 @@ impl RequirementsSpecification {
editables: requirements_txt.editables,
overrides: vec![],
extras: FxHashSet::default(),
index_url: requirements_txt.index_url.map(IndexUrl::from),
extra_index_urls: requirements_txt
.extra_index_urls
.into_iter()
.map(IndexUrl::from)
.collect(),
no_index: requirements_txt.no_index,
find_links: requirements_txt
.find_links
.into_iter()
.map(|link| match link {
FindLink::Url(url) => FlatIndexLocation::Url(url),
FindLink::Path(path) => FlatIndexLocation::Path(path),
})
.collect(),
}
}
RequirementsSource::PyprojectToml(path) => {
@ -151,6 +183,10 @@ impl RequirementsSpecification {
overrides: vec![],
editables: vec![],
extras: used_extras,
index_url: None,
extra_index_urls: vec![],
no_index: false,
find_links: vec![],
}
}
})
@ -176,10 +212,22 @@ impl RequirementsSpecification {
spec.extras.extend(source.extras);
spec.editables.extend(source.editables);
// Use the first project name discovered
// Use the first project name discovered.
if spec.project.is_none() {
spec.project = source.project;
}
if let Some(url) = source.index_url {
if let Some(existing) = spec.index_url {
return Err(anyhow::anyhow!(
"Multiple index URLs specified: `{existing}` vs.` {url}",
));
}
spec.index_url = Some(url);
}
spec.no_index |= source.no_index;
spec.extra_index_urls.extend(source.extra_index_urls);
spec.find_links.extend(source.find_links);
}
// Read all constraints, treating _everything_ as a constraint.
@ -188,6 +236,18 @@ impl RequirementsSpecification {
spec.constraints.extend(source.requirements);
spec.constraints.extend(source.constraints);
spec.constraints.extend(source.overrides);
if let Some(url) = source.index_url {
if let Some(existing) = spec.index_url {
return Err(anyhow::anyhow!(
"Multiple index URLs specified: `{existing}` vs.` {url}",
));
}
spec.index_url = Some(url);
}
spec.no_index |= source.no_index;
spec.extra_index_urls.extend(source.extra_index_urls);
spec.find_links.extend(source.find_links);
}
// Read all overrides, treating both requirements _and_ constraints as overrides.
@ -196,16 +256,25 @@ impl RequirementsSpecification {
spec.overrides.extend(source.requirements);
spec.overrides.extend(source.constraints);
spec.overrides.extend(source.overrides);
if let Some(url) = source.index_url {
if let Some(existing) = spec.index_url {
return Err(anyhow::anyhow!(
"Multiple index URLs specified: `{existing}` vs.` {url}",
));
}
spec.index_url = Some(url);
}
spec.no_index |= source.no_index;
spec.extra_index_urls.extend(source.extra_index_urls);
spec.find_links.extend(source.find_links);
}
Ok(spec)
}
/// Read the requirements from a set of sources.
pub(crate) fn requirements_and_editables(
requirements: &[RequirementsSource],
) -> Result<(Vec<Requirement>, Vec<EditableRequirement>)> {
let specification = Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None)?;
Ok((specification.requirements, specification.editables))
pub(crate) fn from_simple_sources(requirements: &[RequirementsSource]) -> Result<Self> {
Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None)
}
}

161
crates/puffin/tests/pip_compile.rs
View file

@ -3350,6 +3350,49 @@ fn find_links_url() -> Result<()> {
Ok(())
}
/// Compile using `--find-links` with a URL by resolving `tqdm` from the `PyTorch` wheels index,
/// with the URL itself provided in a `requirements.txt` file.
#[test]
fn find_links_requirements_txt() -> Result<()> {
let temp_dir = TempDir::new()?;
let cache_dir = TempDir::new()?;
let venv = create_venv_py312(&temp_dir, &cache_dir);
let requirements_in = temp_dir.child("requirements.in");
requirements_in.write_str("-f https://download.pytorch.org/whl/torch_stable.html\ntqdm")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip")
.arg("compile")
.arg("requirements.in")
.arg("--no-index")
.arg("--emit-find-links")
.arg("--cache-dir")
.arg(cache_dir.path())
.arg("--exclude-newer")
.arg(EXCLUDE_NEWER)
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by Puffin v[VERSION] via the following command:
# puffin pip compile requirements.in --no-index --emit-find-links --cache-dir [CACHE_DIR]
--find-links https://download.pytorch.org/whl/torch_stable.html
tqdm==4.64.1
----- stderr -----
Resolved 1 package in [TIME]
"###);
});
Ok(())
}
/// Use an existing resolution for `black==23.10.1`, with stale versions of `click` and `pathspec`.
/// Nothing should change.
#[test]
@ -3689,7 +3732,7 @@ fn missing_package_name() -> Result<()> {
----- stdout -----
----- stderr -----
error: Unsupported requirement in requirements.in position 0 to 135
error: Unsupported requirement in requirements.in at position 0
Caused by: URL requirement must be preceded by a package name. Add the name of the package before the URL (e.g., `package_name @ https://...`).
https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -3974,3 +4017,119 @@ fn emit_find_links() -> Result<()> {
Ok(())
}
/// Respect the `--no-index` flag in a `requirements.txt` file.
#[test]
fn no_index_requirements_txt() -> Result<()> {
let temp_dir = TempDir::new()?;
let cache_dir = TempDir::new()?;
let venv = create_venv_py312(&temp_dir, &cache_dir);
let requirements_in = temp_dir.child("requirements.in");
requirements_in.write_str("--no-index\ntqdm")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip")
.arg("compile")
.arg("requirements.in")
.arg("--cache-dir")
.arg(cache_dir.path())
.arg("--exclude-newer")
.arg(EXCLUDE_NEWER)
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir), @r###"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: tqdm isn't available locally, but making network requests to registries was banned.
"###);
});
Ok(())
}
/// Prefer the `--index-url` from the command line over the `--index-url` in a `requirements.txt`
/// file.
#[test]
fn index_url_requirements_txt() -> Result<()> {
let temp_dir = TempDir::new()?;
let cache_dir = TempDir::new()?;
let venv = create_venv_py312(&temp_dir, &cache_dir);
let requirements_in = temp_dir.child("requirements.in");
requirements_in.write_str("--index-url https://google.com\ntqdm")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip")
.arg("compile")
.arg("requirements.in")
.arg("--index-url")
.arg("https://pypi.org/simple")
.arg("--cache-dir")
.arg(cache_dir.path())
.arg("--exclude-newer")
.arg(EXCLUDE_NEWER)
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by Puffin v[VERSION] via the following command:
# puffin pip compile requirements.in --index-url https://pypi.org/simple --cache-dir [CACHE_DIR]
tqdm==4.66.1
----- stderr -----
Resolved 1 package in [TIME]
"###);
});
Ok(())
}
/// Raise an error when multiple `requirements.txt` files include `--index-url` flags.
#[test]
fn conflicting_index_urls_requirements_txt() -> Result<()> {
let temp_dir = TempDir::new()?;
let cache_dir = TempDir::new()?;
let venv = create_venv_py312(&temp_dir, &cache_dir);
let requirements_in = temp_dir.child("requirements.in");
requirements_in.write_str("--index-url https://google.com\ntqdm")?;
let constraints_in = temp_dir.child("constraints.in");
constraints_in.write_str("--index-url https://wikipedia.org\nflask")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip")
.arg("compile")
.arg("requirements.in")
.arg("--constraint")
.arg("constraints.in")
.arg("--cache-dir")
.arg(cache_dir.path())
.arg("--exclude-newer")
.arg(EXCLUDE_NEWER)
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir), @r###"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: Multiple index URLs specified: `https://google.com/` vs.` https://wikipedia.org/
"###);
});
Ok(())
}

205
crates/requirements-txt/src/lib.rs
View file

@ -45,7 +45,7 @@ use unscanny::{Pattern, Scanner};
use url::Url;
use pep508_rs::{split_scheme, Extras, Pep508Error, Pep508ErrorSource, Requirement, VerbatimUrl};
use puffin_fs::NormalizedDisplay;
use puffin_fs::{normalize_url_path, NormalizedDisplay};
use puffin_normalize::ExtraName;
/// We emit one of those for each requirements.txt entry
@ -66,6 +66,62 @@ enum RequirementsTxtStatement {
RequirementEntry(RequirementEntry),
/// `-e`
EditableRequirement(EditableRequirement),
/// `--index-url`
IndexUrl(Url),
/// `--extra-index-url`
ExtraIndexUrl(Url),
/// `--find-links`
FindLinks(FindLink),
/// `--no-index`
NoIndex,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FindLink {
Path(PathBuf),
Url(Url),
}
impl FindLink {
/// Parse a raw string for a `--find-links` entry, which could be a URL or a local path.
///
/// For example:
/// - `file:///home/ferris/project/scripts/...`
/// - `file:../ferris/`
/// - `../ferris/`
/// - `https://download.pytorch.org/whl/torch_stable.html`
pub fn parse(given: &str, working_dir: impl AsRef<Path>) -> Result<Self, url::ParseError> {
if let Some((scheme, path)) = split_scheme(given) {
if scheme == "file" {
// Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/`
let path = path.strip_prefix("//").unwrap_or(path);
// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);
let path = PathBuf::from(path.as_ref());
let path = if path.is_absolute() {
path
} else {
working_dir.as_ref().join(path)
};
Ok(Self::Path(path))
} else {
// Ex) `https://download.pytorch.org/whl/torch_stable.html`
let url = Url::parse(given)?;
Ok(Self::Url(url))
}
} else {
// Ex) `../ferris/`
let path = PathBuf::from(given);
let path = if path.is_absolute() {
path
} else {
working_dir.as_ref().join(path)
};
Ok(Self::Path(path))
}
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
@ -134,15 +190,15 @@ impl EditableRequirement {
// Create a `VerbatimUrl` to represent the editable requirement.
let url = if let Some((scheme, path)) = split_scheme(requirement) {
if scheme == "file" {
if let Some(path) = path.strip_prefix("//") {
// Ex) `file:///home/ferris/project/scripts/...`
VerbatimUrl::from_path(path, working_dir.as_ref())
} else {
// Ex) `file:../editable/`
VerbatimUrl::from_path(path, working_dir.as_ref())
}
// Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/`
let path = path.strip_prefix("//").unwrap_or(path);
// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);
VerbatimUrl::from_path(path, working_dir.as_ref())
} else {
// Ex) `https://...`
// Ex) `https://download.pytorch.org/whl/torch_stable.html`
return Err(RequirementsTxtParserError::UnsupportedUrl(
requirement.to_string(),
));
@ -218,14 +274,22 @@ impl Display for RequirementEntry {
}
/// Parsed and flattened requirements.txt with requirements and constraints
#[derive(Debug, Clone, Default, Eq, PartialEq)]
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct RequirementsTxt {
/// The actual requirements with the hashes
/// The actual requirements with the hashes.
pub requirements: Vec<RequirementEntry>,
/// Constraints included with `-c`
/// Constraints included with `-c`.
pub constraints: Vec<Requirement>,
/// Editables with `-e`
/// Editables with `-e`.
pub editables: Vec<EditableRequirement>,
/// The index URL, specified with `--index-url`.
pub index_url: Option<Url>,
/// The extra index URLs, specified with `--extra-index-url`.
pub extra_index_urls: Vec<Url>,
/// The find links locations, specified with `--find-links`.
pub find_links: Vec<FindLink>,
/// Whether to ignore the index, specified with `--no-index`.
pub no_index: bool,
}
impl RequirementsTxt {
@ -313,6 +377,24 @@ impl RequirementsTxt {
RequirementsTxtStatement::EditableRequirement(editable) => {
data.editables.push(editable);
}
RequirementsTxtStatement::IndexUrl(url) => {
if data.index_url.is_some() {
return Err(RequirementsTxtParserError::Parser {
message: "Multiple `--index-url` values provided".to_string(),
location: s.cursor(),
});
}
data.index_url = Some(url);
}
RequirementsTxtStatement::ExtraIndexUrl(url) => {
data.extra_index_urls.push(url);
}
RequirementsTxtStatement::FindLinks(path_or_url) => {
data.find_links.push(path_or_url);
}
RequirementsTxtStatement::NoIndex => {
data.no_index = true;
}
}
}
Ok(data)
@ -366,6 +448,37 @@ fn parse_entry(
let editable_requirement = EditableRequirement::parse(path_or_url, working_dir)
.map_err(|err| err.with_offset(start))?;
RequirementsTxtStatement::EditableRequirement(editable_requirement)
} else if s.eat_if("-i") || s.eat_if("--index-url") {
let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
source: err,
url: url.to_string(),
start,
end: s.cursor(),
})?;
RequirementsTxtStatement::IndexUrl(url)
} else if s.eat_if("--extra-index-url") {
let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url {
source: err,
url: url.to_string(),
start,
end: s.cursor(),
})?;
RequirementsTxtStatement::ExtraIndexUrl(url)
} else if s.eat_if("--no-index") {
RequirementsTxtStatement::NoIndex
} else if s.eat_if("--find-links") || s.eat_if("-f") {
let path_or_url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?;
let path_or_url = FindLink::parse(path_or_url, working_dir).map_err(|err| {
RequirementsTxtParserError::Url {
source: err,
url: path_or_url.to_string(),
start,
end: s.cursor(),
}
})?;
RequirementsTxtStatement::FindLinks(path_or_url)
} else if s.at(char::is_ascii_alphanumeric) {
let (requirement, hashes) = parse_requirement_and_hashes(s, content, working_dir)?;
RequirementsTxtStatement::RequirementEntry(RequirementEntry {
@ -545,6 +658,12 @@ pub struct RequirementsTxtFileError {
#[derive(Debug)]
pub enum RequirementsTxtParserError {
IO(io::Error),
Url {
source: url::ParseError,
url: String,
start: usize,
end: usize,
},
InvalidEditablePath(String),
UnsupportedUrl(String),
Parser {
@ -577,6 +696,17 @@ impl RequirementsTxtParserError {
RequirementsTxtParserError::InvalidEditablePath(given) => {
RequirementsTxtParserError::InvalidEditablePath(given)
}
RequirementsTxtParserError::Url {
source,
url,
start,
end,
} => RequirementsTxtParserError::Url {
source,
url,
start: start + offset,
end: end + offset,
},
RequirementsTxtParserError::UnsupportedUrl(url) => {
RequirementsTxtParserError::UnsupportedUrl(url)
}
@ -618,8 +748,11 @@ impl Display for RequirementsTxtParserError {
RequirementsTxtParserError::InvalidEditablePath(given) => {
write!(f, "Invalid editable path: {given}")
}
RequirementsTxtParserError::Url { url, start, .. } => {
write!(f, "Invalid URL at position {start}: `{url}`")
}
RequirementsTxtParserError::UnsupportedUrl(url) => {
write!(f, "Unsupported URL (expected a `file://` scheme): {url}")
write!(f, "Unsupported URL (expected a `file://` scheme): `{url}`")
}
RequirementsTxtParserError::Parser { message, location } => {
write!(f, "{message} at position {location}")
@ -641,6 +774,7 @@ impl std::error::Error for RequirementsTxtParserError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match &self {
RequirementsTxtParserError::IO(err) => err.source(),
RequirementsTxtParserError::Url { source, .. } => Some(source),
RequirementsTxtParserError::InvalidEditablePath(_) => None,
RequirementsTxtParserError::UnsupportedUrl(_) => None,
RequirementsTxtParserError::UnsupportedRequirement { source, .. } => Some(source),
@ -655,6 +789,13 @@ impl Display for RequirementsTxtFileError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match &self.error {
RequirementsTxtParserError::IO(err) => err.fmt(f),
RequirementsTxtParserError::Url { url, start, .. } => {
write!(
f,
"Invalid URL in `{}` at position {start}: `{url}`",
self.file.normalized_display(),
)
}
RequirementsTxtParserError::InvalidEditablePath(given) => {
write!(
f,
@ -665,7 +806,7 @@ impl Display for RequirementsTxtFileError {
RequirementsTxtParserError::UnsupportedUrl(url) => {
write!(
f,
"Unsupported URL (expected a `file://` scheme) in `{}`: {url}",
"Unsupported URL (expected a `file://` scheme) in `{}`: `{url}`",
self.file.normalized_display(),
)
}
@ -676,13 +817,11 @@ impl Display for RequirementsTxtFileError {
self.file.normalized_display(),
)
}
RequirementsTxtParserError::UnsupportedRequirement { start, end, .. } => {
RequirementsTxtParserError::UnsupportedRequirement { start, .. } => {
write!(
f,
"Unsupported requirement in {} position {} to {}",
"Unsupported requirement in {} at position {start}",
self.file.normalized_display(),
start,
end,
)
}
RequirementsTxtParserError::Pep508 { start, .. } => {
@ -865,7 +1004,7 @@ mod test {
insta::with_settings!({
filters => vec![(requirements_txt.path().to_str().unwrap(), "<REQUIREMENTS_TXT>")]
}, {
insta::assert_display_snapshot!(errors, @"Unsupported URL (expected a `file://` scheme) in `<REQUIREMENTS_TXT>`: http://localhost:8080/");
insta::assert_display_snapshot!(errors, @"Unsupported URL (expected a `file://` scheme) in `<REQUIREMENTS_TXT>`: `http://localhost:8080/`");
});
Ok(())
@ -899,6 +1038,32 @@ mod test {
Ok(())
}
#[test]
fn invalid_index_url() -> Result<()> {
let temp_dir = assert_fs::TempDir::new()?;
let requirements_txt = temp_dir.child("requirements.txt");
requirements_txt.write_str(indoc! {"
--index-url 123
"})?;
let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err();
let errors = anyhow::Error::new(error)
.chain()
.map(|err| err.to_string().replace('\\', "/"))
.join("\n");
insta::with_settings!({
filters => vec![(requirements_txt.path().to_str().unwrap(), "<REQUIREMENTS_TXT>")]
}, {
insta::assert_display_snapshot!(errors, @r###"
Invalid URL in `<REQUIREMENTS_TXT>` at position 0: `123`
relative URL without a base
"###);
});
Ok(())
}
#[test]
fn editable_extra() {
assert_eq!(

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap
View file

@ -145,4 +145,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap
View file

@ -69,4 +69,8 @@ RequirementsTxt {
},
],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap
View file

@ -53,4 +53,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap
View file

@ -58,4 +58,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap
View file

@ -6,4 +6,8 @@ RequirementsTxt {
requirements: [],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap
View file

@ -96,4 +96,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap
View file

@ -42,4 +42,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap
View file

@ -19,4 +19,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap
View file

@ -282,4 +282,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap
View file

@ -53,4 +53,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap
View file

@ -58,4 +58,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap
View file

@ -145,4 +145,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap
View file

@ -69,4 +69,8 @@ RequirementsTxt {
},
],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap
View file

@ -53,4 +53,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap
View file

@ -6,4 +6,8 @@ RequirementsTxt {
requirements: [],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap
View file

@ -96,4 +96,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap
View file

@ -42,4 +42,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap
View file

@ -19,4 +19,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap
View file

@ -282,4 +282,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap
View file

@ -53,4 +53,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap
View file

@ -58,4 +58,8 @@ RequirementsTxt {
],
constraints: [],
editables: [],
index_url: None,
extra_index_urls: [],
find_links: [],
no_index: false,
}