mirrors/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv.git synced 2025-07-24 05:35:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Filter out incompatible dists (#398)

Browse source 
Filter out source dists and wheels whose `requires-python` from the
simple api is incompatible with the current python version.

This change showed an important problem: When we use a fake python
version for resolving, building source distributions breaks down because
we can only build with versions we actually have.

This change became surprisingly big. The tests now require python 3.7 to
be installed, but changing that would mean an even bigger change.

Fixes #388
This commit is contained in:
konsti 2023-11-13 17:14:07 +01:00 • committed by GitHub
parent 81c9cd0d4a
commit 76a41066ac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 365 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/ci.yaml vendored
View file

@ -48,10 +48,13 @@ jobs:
name: "cargo test | ${{ matrix.os }}"
steps:
- uses: actions/checkout@v4
# TODO(konstin): Mock the venv in the installer test so we don't need 3.8 anymore
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
python-version: |
3.8
${{ env.PYTHON_VERSION }}
- name: "Install Rust toolchain"
run: rustup show
- name: "Install cargo insta"

3
.gitignore vendored
View file

@ -13,3 +13,6 @@ target/
# Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations
cache-*
# python tmp files
__pycache__

10
crates/pep440-rs/src/version_specifier.rs
View file

@ -540,6 +540,9 @@ impl Display for VersionSpecifier {
/// ```
pub fn parse_version_specifiers(spec: &str) -> Result<Vec<VersionSpecifier>, Pep440Error> {
let mut version_ranges = Vec::new();
if spec.is_empty() {
return Ok(version_ranges);
}
let mut start: usize = 0;
let separator = ",";
for version_range_spec in spec.split(separator) {
@ -1301,4 +1304,11 @@ mod test {
">=3.7, <4.0, !=3.9.0"
);
}
/// These occur in the simple api, e.g.
/// <https://pypi.org/simple/geopandas/?format=application/vnd.pypi.simple.v1+json>
#[test]
fn test_version_specifiers_empty() {
assert_eq!(VersionSpecifiers::from_str("").unwrap().to_string(), "");
}
}

7
crates/puffin-cli/src/commands/pip_compile.rs
View file

@ -125,6 +125,11 @@ pub(crate) async fn pip_compile(
|| Cow::Borrowed(venv.interpreter_info().markers()),
|python_version| Cow::Owned(python_version.markers(venv.interpreter_info().markers())),
);
// Inject the fake python version if necessary
let interpreter_info = venv
.interpreter_info()
.clone()
.patch_markers(markers.clone().into_owned());
// Instantiate a client.
let client = {
@ -143,7 +148,7 @@ pub(crate) async fn pip_compile(
let build_dispatch = BuildDispatch::new(
client.clone(),
cache.to_path_buf(),
venv.interpreter_info().clone(),
interpreter_info,
fs::canonicalize(venv.python_executable())?,
no_build,
);

3
crates/puffin-cli/src/commands/pip_sync.rs
View file

@ -128,7 +128,8 @@ pub(crate) async fn sync_requirements(
} else {
let start = std::time::Instant::now();
let wheel_finder = puffin_resolver::DistFinder::new(&tags, &client)
let wheel_finder =
puffin_resolver::DistFinder::new(&tags, &client, venv.interpreter_info())
.with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64));
let resolution = wheel_finder.resolve(&remote).await?;

49
crates/puffin-cli/tests/pip_compile.rs
View file

@ -630,6 +630,55 @@ fn compile_python_37() -> Result<()> {
Ok(())
}
/// Test that we select the last 3.8 compatible numpy version instead of trying to compile an
/// incompatible sdist <https://github.com/astral-sh/puffin/issues/388>
#[test]
fn compile_numpy_py38() -> Result<()> {
let temp_dir = assert_fs::TempDir::new()?;
let cache_dir = assert_fs::TempDir::new()?;
let venv = temp_dir.child(".venv");
Command::new(get_cargo_bin(BIN_NAME))
.arg("venv")
.arg(venv.as_os_str())
.arg("--cache-dir")
.arg(cache_dir.path())
.current_dir(&temp_dir)
.assert()
.success();
venv.assert(predicates::path::is_dir());
let requirements_in = temp_dir.child("requirements.in");
requirements_in.touch()?;
requirements_in.write_str("numpy")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip-compile")
.arg("requirements.in")
.arg("--python-version")
.arg("py38")
.arg("--cache-dir")
.arg(cache_dir.path())
// Check that we select the wheel and not the sdist
.arg("--no-build")
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir), @r###"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: Failed to build distribution: numpy-1.24.4.tar.gz
Caused by: Building source distributions is disabled
"###);
});
Ok(())
}
/// Resolve a specific Flask wheel via a URL dependency.
#[test]
fn compile_wheel_url_dependency() -> Result<()> {

49
crates/puffin-cli/tests/pip_sync.rs
View file

@ -2,7 +2,7 @@
use std::process::Command;
use anyhow::Result;
use anyhow::{Context, Result};
use assert_cmd::prelude::*;
use assert_fs::prelude::*;
use insta_cmd::_macro_support::insta;
@ -928,3 +928,50 @@ fn install_version_then_install_url() -> Result<()> {
Ok(())
}
/// Test that we select the last 3.8 compatible numpy version instead of trying to compile an
/// incompatible sdist <https://github.com/astral-sh/puffin/issues/388>
#[test]
fn install_numpy_py38() -> Result<()> {
let temp_dir = assert_fs::TempDir::new()?;
let cache_dir = assert_fs::TempDir::new()?;
let venv = temp_dir.child(".venv");
Command::new(get_cargo_bin(BIN_NAME))
.arg("venv")
.arg(venv.as_os_str())
.arg("--python")
// TODO(konstin): Mock the venv in the installer test so we don't need this anymore
.arg(which::which("python3.8").context("python3.8 must be installed")?)
.arg("--cache-dir")
.arg(cache_dir.path())
.current_dir(&temp_dir)
.assert()
.success();
venv.assert(predicates::path::is_dir());
let requirements_txt = temp_dir.child("requirements.txt");
requirements_txt.touch()?;
requirements_txt.write_str("numpy")?;
insta::with_settings!({
filters => INSTA_FILTERS.to_vec()
}, {
assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
.arg("pip-sync")
.arg("requirements.txt")
.arg("--cache-dir")
.arg(cache_dir.path())
.env("VIRTUAL_ENV", venv.as_os_str())
.current_dir(&temp_dir));
});
Command::new(venv.join("bin").join("python"))
.arg("-c")
.arg("import numpy")
.current_dir(&temp_dir)
.assert()
.success();
Ok(())
}

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_constraints_markers.snap
View file

@ -8,9 +8,9 @@ info:
- "--constraint"
- constraints.txt
- "--cache-dir"
- /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1Ts53o
- /tmp/.tmp81zzAa
env:
VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1vzBQa/.venv
VIRTUAL_ENV: /tmp/.tmpFXMFLd/.venv
---
success: true
exit_code: 0

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_constraints_txt.snap
View file

@ -8,9 +8,9 @@ info:
- "--constraint"
- constraints.txt
- "--cache-dir"
- /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpvjdHOb
- /tmp/.tmpuS4Jn4
env:
VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpSAaWi3/.venv
VIRTUAL_ENV: /tmp/.tmpnFovuD/.venv
---
success: true
exit_code: 0

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml.snap
View file

@ -6,9 +6,9 @@ info:
- pip-compile
- pyproject.toml
- "--cache-dir"
- /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpOOTFwj
- /tmp/.tmpWzFzu7
env:
VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpU0VXyY/.venv
VIRTUAL_ENV: /tmp/.tmpSiRk6d/.venv
---
success: true
exit_code: 0

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_all_extras.snap
View file

@ -7,9 +7,9 @@ info:
- pyproject.toml
- "--all-extras"
- "--cache-dir"
- /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpFzJKRe
- /tmp/.tmpQHVFqr
env:
VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpdadcmu/.venv
VIRTUAL_ENV: /tmp/.tmpmKXqaF/.venv
---
success: true
exit_code: 0

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_extra.snap
View file

@ -8,9 +8,9 @@ info:
- "--extra"
- foo
- "--cache-dir"
- /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpAYEAdM
- /tmp/.tmpLjFVr0
env:
VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1xuOcV/.venv
VIRTUAL_ENV: /tmp/.tmpDT0oRC/.venv
---
success: true
exit_code: 0

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_extra_name_normalization.snap
View file

@ -8,9 +8,9 @@ info:
- "--extra"
- FRiENDlY-...-_-BARd
- "--cache-dir"
- /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpFyLXQn
- /tmp/.tmpga6JO1
env:
VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpxfZatv/.venv
VIRTUAL_ENV: /tmp/.tmpwmJIym/.venv
---
success: true
exit_code: 0

36
crates/puffin-cli/tests/snapshots/pip_compile__compile_python_37.snap
View file

@ -8,38 +8,16 @@ info:
- "--python-version"
- py37
- "--cache-dir"
- /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpzwzUVe
- /tmp/.tmpHz7Dc2
env:
VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpqFv4YL/.venv
VIRTUAL_ENV: /tmp/.tmpoMjDJT/.venv
---
success: true
exit_code: 0
success: false
exit_code: 1
----- stdout -----
# This file was autogenerated by Puffin v0.0.1 via the following command:
# [BIN_PATH] pip-compile requirements.in --python-version py37 --cache-dir [CACHE_DIR]
black==23.10.1
click==8.1.7
# via black
importlib-metadata==6.8.0
# via click
mypy-extensions==1.0.0
# via black
packaging==23.2
# via black
pathspec==0.11.2
# via black
platformdirs==4.0.0
# via black
tomli==2.0.1
# via black
typing-extensions==4.8.0
# via
# black
# importlib-metadata
# platformdirs
zipp==3.17.0
# via importlib-metadata
----- stderr -----
Resolved 10 packages in [TIME]
× No solution found when resolving dependencies:
╰─▶ Because there is no version of black available matching ==23.10.1 and
root depends on black==23.10.1, version solving failed.

4
crates/puffin-cli/tests/snapshots/pip_compile__compile_requirements_in.snap
View file

@ -6,9 +6,9 @@ info:
- pip-compile
- requirements.in
- "--cache-dir"
- /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpxF1zFY
- /tmp/.tmpQB4Ze4
env:
VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmp33QZdv/.venv
VIRTUAL_ENV: /tmp/.tmpSYcMgG/.venv
---
success: true
exit_code: 0

23
crates/puffin-cli/tests/snapshots/pip_sync__install_numpy_py38.snap Normal file
View file

@ -0,0 +1,23 @@
---
source: crates/puffin-cli/tests/pip_sync.rs
info:
program: puffin
args:
- pip-sync
- requirements.txt
- "--cache-dir"
- /tmp/.tmpL9E9fl
env:
VIRTUAL_ENV: /tmp/.tmp0yDHj5/.venv
---
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Downloaded 1 package in [TIME]
Unzipped 1 package in [TIME]
Installed 1 package in [TIME]
+ numpy==1.24.4

2
crates/puffin-dispatch/src/lib.rs
View file

@ -135,7 +135,7 @@ impl BuildContext for BuildDispatch {
if remote.len() == 1 { "" } else { "s" },
remote.iter().map(ToString::to_string).join(", ")
);
let resolution = DistFinder::new(&tags, &self.client)
let resolution = DistFinder::new(&tags, &self.client, self.interpreter_info())
.resolve(&remote)
.await
.context("Failed to resolve build dependencies")?;

4
crates/puffin-distribution/src/traits.rs
View file

@ -12,8 +12,8 @@ pub trait Metadata {
/// Return the normalized [`PackageName`] of the distribution.
fn name(&self) -> &PackageName;
/// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based
/// distributions.
/// Return a [`pep440_rs::Version`], for registry-based distributions, or a [`url::Url`],
/// for URL-based distributions.
fn version_or_url(&self) -> VersionOrUrl;
/// Returns a unique identifier for the package.

21
crates/puffin-interpreter/src/interpreter_info.rs
View file

@ -44,6 +44,21 @@ impl InterpreterInfo {
base_prefix: info.base_prefix,
})
}
// TODO(konstin): Find a better way mocking the fields
pub fn artificial(
platform: Platform,
markers: MarkerEnvironment,
base_exec_prefix: PathBuf,
base_prefix: PathBuf,
) -> Self {
Self {
platform: PythonPlatform(platform),
markers,
base_exec_prefix,
base_prefix,
}
}
}
impl InterpreterInfo {
@ -76,6 +91,12 @@ impl InterpreterInfo {
pub fn base_prefix(&self) -> &Path {
&self.base_prefix
}
/// Inject markers of a fake python version
#[must_use]
pub fn patch_markers(self, markers: MarkerEnvironment) -> Self {
Self { markers, ..self }
}
}
#[derive(Debug, Error)]

1
crates/puffin-resolver/Cargo.toml
View file

@ -21,6 +21,7 @@ puffin-cache = { path = "../puffin-cache" }
puffin-client = { path = "../puffin-client" }
puffin-distribution = { path = "../puffin-distribution" }
puffin-git = { path = "../puffin-git" }
puffin-interpreter = { path = "../puffin-interpreter" }
puffin-normalize = { path = "../puffin-normalize" }
puffin-traits = { path = "../puffin-traits" }
pypi-types = { path = "../pypi-types" }

23
crates/puffin-resolver/src/finder.rs
View file

@ -14,6 +14,7 @@ use pep508_rs::{Requirement, VersionOrUrl};
use platform_tags::Tags;
use puffin_client::RegistryClient;
use puffin_distribution::Dist;
use puffin_interpreter::InterpreterInfo;
use puffin_normalize::PackageName;
use pypi_types::{File, SimpleJson};
@ -24,15 +25,21 @@ pub struct DistFinder<'a> {
tags: &'a Tags,
client: &'a RegistryClient,
reporter: Option<Box<dyn Reporter>>,
interpreter_info: &'a InterpreterInfo,
}
impl<'a> DistFinder<'a> {
/// Initialize a new distribution finder.
pub fn new(tags: &'a Tags, client: &'a RegistryClient) -> Self {
pub fn new(
tags: &'a Tags,
client: &'a RegistryClient,
interpreter_info: &'a InterpreterInfo,
) -> Self {
Self {
tags,
client,
reporter: None,
interpreter_info,
}
}
@ -129,6 +136,20 @@ impl<'a> DistFinder<'a> {
fn select(&self, requirement: &Requirement, files: Vec<File>) -> Option<Dist> {
let mut fallback = None;
for file in files.into_iter().rev() {
// Only add dists compatible with the python version.
// This is relevant for source dists which give no other indication of their
// compatibility and wheels which may be tagged `py3-none-any` but
// have `requires-python: ">=3.9"`
// TODO(konstin): https://github.com/astral-sh/puffin/issues/406
if !file
.requires_python
.as_ref()
.map_or(true, |requires_python| {
requires_python.contains(self.interpreter_info.version())
})
{
continue;
}
if let Ok(wheel) = WheelFilename::from_str(file.filename.as_str()) {
if !wheel.is_compatible(self.tags) {
continue;

15
crates/puffin-resolver/src/resolver.rs
View file

@ -536,6 +536,21 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> {
// distributions.
let mut version_map: VersionMap = BTreeMap::new();
for file in metadata.files {
// Only add dists compatible with the python version.
// This is relevant for source dists which give no other indication of their
// compatibility and wheels which may be tagged `py3-none-any` but
// have `requires-python: ">=3.9"`
// TODO(konstin): https://github.com/astral-sh/puffin/issues/406
if !file
.requires_python
.as_ref()
.map_or(true, |requires_python| {
requires_python
.contains(self.build_context.interpreter_info().version())
})
{
continue;
}
if let Ok(filename) = WheelFilename::from_str(file.filename.as_str()) {
if filename.is_compatible(self.tags) {
let version = PubGrubVersion::from(filename.version.clone());

131
crates/puffin-resolver/tests/resolver.rs
View file

@ -4,7 +4,7 @@
//! `PyPI` directly.
use std::future::Future;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::str::FromStr;
@ -17,10 +17,12 @@ use platform_host::{Arch, Os, Platform};
use platform_tags::Tags;
use puffin_client::RegistryClientBuilder;
use puffin_interpreter::{InterpreterInfo, Virtualenv};
use puffin_resolver::{Manifest, PreReleaseMode, ResolutionMode, Resolver};
use puffin_resolver::{Graph, Manifest, PreReleaseMode, ResolutionMode, Resolver};
use puffin_traits::BuildContext;
struct DummyContext;
struct DummyContext {
interpreter_info: InterpreterInfo,
}
impl BuildContext for DummyContext {
fn cache(&self) -> &Path {
@ -28,7 +30,7 @@ impl BuildContext for DummyContext {
}
fn interpreter_info(&self) -> &InterpreterInfo {
panic!("The test should not need to build source distributions")
&self.interpreter_info
}
fn base_python(&self) -> &Path {
@ -61,13 +63,29 @@ impl BuildContext for DummyContext {
}
}
async fn resolve(
manifest: Manifest,
markers: &'static MarkerEnvironment,
tags: &Tags,
) -> Result<Graph> {
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let build_context = DummyContext {
interpreter_info: InterpreterInfo::artificial(
Platform::current()?,
markers.clone(),
PathBuf::from("/dev/null"),
PathBuf::from("/dev/null"),
),
};
let resolver = Resolver::new(manifest, markers, tags, &client, &build_context);
Ok(resolver.resolve().await?)
}
#[tokio::test]
async fn black() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![],
@ -77,8 +95,7 @@ async fn black() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -89,9 +106,6 @@ async fn black() -> Result<()> {
async fn black_colorama() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black[colorama]<=23.9.1").unwrap()],
vec![],
@ -101,8 +115,7 @@ async fn black_colorama() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -113,9 +126,6 @@ async fn black_colorama() -> Result<()> {
async fn black_python_310() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![],
@ -125,8 +135,7 @@ async fn black_python_310() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_310, &TAGS_310, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_310, &TAGS_310).await?;
insta::assert_display_snapshot!(resolution);
@ -139,9 +148,6 @@ async fn black_python_310() -> Result<()> {
async fn black_mypy_extensions() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![Requirement::from_str("mypy-extensions<0.4.4").unwrap()],
@ -151,8 +157,7 @@ async fn black_mypy_extensions() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -165,9 +170,6 @@ async fn black_mypy_extensions() -> Result<()> {
async fn black_mypy_extensions_extra() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![Requirement::from_str("mypy-extensions[extra]<0.4.4").unwrap()],
@ -177,8 +179,7 @@ async fn black_mypy_extensions_extra() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -191,9 +192,6 @@ async fn black_mypy_extensions_extra() -> Result<()> {
async fn black_flake8() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![Requirement::from_str("flake8<1").unwrap()],
@ -203,8 +201,7 @@ async fn black_flake8() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -215,9 +212,6 @@ async fn black_flake8() -> Result<()> {
async fn black_lowest() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black>21").unwrap()],
vec![],
@ -227,8 +221,7 @@ async fn black_lowest() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -239,9 +232,6 @@ async fn black_lowest() -> Result<()> {
async fn black_lowest_direct() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black>21").unwrap()],
vec![],
@ -251,8 +241,7 @@ async fn black_lowest_direct() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -263,9 +252,6 @@ async fn black_lowest_direct() -> Result<()> {
async fn black_respect_preference() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![],
@ -275,8 +261,7 @@ async fn black_respect_preference() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -287,9 +272,6 @@ async fn black_respect_preference() -> Result<()> {
async fn black_ignore_preference() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=23.9.1").unwrap()],
vec![],
@ -299,8 +281,7 @@ async fn black_ignore_preference() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -311,9 +292,6 @@ async fn black_ignore_preference() -> Result<()> {
async fn black_disallow_prerelease() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=20.0").unwrap()],
vec![],
@ -323,8 +301,9 @@ async fn black_disallow_prerelease() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let err = resolver.resolve().await.unwrap_err();
let err = resolve(manifest, &MARKERS_311, &TAGS_311)
.await
.unwrap_err();
insta::assert_display_snapshot!(err);
@ -335,9 +314,6 @@ async fn black_disallow_prerelease() -> Result<()> {
async fn black_allow_prerelease_if_necessary() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("black<=20.0").unwrap()],
vec![],
@ -347,10 +323,11 @@ async fn black_allow_prerelease_if_necessary() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await.unwrap_err();
let err = resolve(manifest, &MARKERS_311, &TAGS_311)
.await
.unwrap_err();
insta::assert_display_snapshot!(resolution);
insta::assert_display_snapshot!(err);
Ok(())
}
@ -359,9 +336,6 @@ async fn black_allow_prerelease_if_necessary() -> Result<()> {
async fn pylint_disallow_prerelease() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("pylint==2.3.0").unwrap()],
vec![],
@ -371,8 +345,7 @@ async fn pylint_disallow_prerelease() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -383,9 +356,6 @@ async fn pylint_disallow_prerelease() -> Result<()> {
async fn pylint_allow_prerelease() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![Requirement::from_str("pylint==2.3.0").unwrap()],
vec![],
@ -395,8 +365,7 @@ async fn pylint_allow_prerelease() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -407,9 +376,6 @@ async fn pylint_allow_prerelease() -> Result<()> {
async fn pylint_allow_explicit_prerelease_without_marker() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![
Requirement::from_str("pylint==2.3.0").unwrap(),
@ -422,8 +388,7 @@ async fn pylint_allow_explicit_prerelease_without_marker() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);
@ -434,9 +399,6 @@ async fn pylint_allow_explicit_prerelease_without_marker() -> Result<()> {
async fn pylint_allow_explicit_prerelease_with_marker() -> Result<()> {
colored::control::set_override(false);
let tempdir = tempdir()?;
let client = RegistryClientBuilder::new(tempdir.path()).build();
let manifest = Manifest::new(
vec![
Requirement::from_str("pylint==2.3.0").unwrap(),
@ -449,8 +411,7 @@ async fn pylint_allow_explicit_prerelease_with_marker() -> Result<()> {
None,
);
let resolver = Resolver::new(manifest, &MARKERS_311, &TAGS_311, &client, &DummyContext);
let resolution = resolver.resolve().await?;
let resolution = resolve(manifest, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution);

64
crates/pypi-types/src/lenient_requirement.rs Normal file
View file

@ -0,0 +1,64 @@
use pep440_rs::{Pep440Error, VersionSpecifiers};
use serde::{de, Deserialize, Deserializer, Serialize};
use std::str::FromStr;
use tracing::warn;
/// Like [`VersionSpecifiers`], but attempts to correct some common errors in user-provided requirements.
///
/// We turn `>=3.x.*` into `>=3.x`
#[derive(Debug, Clone, Serialize, Eq, PartialEq)]
pub struct LenientVersionSpecifiers(VersionSpecifiers);
impl FromStr for LenientVersionSpecifiers {
type Err = Pep440Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match VersionSpecifiers::from_str(s) {
Ok(specifiers) => Ok(Self(specifiers)),
Err(err) => {
// Given `>=3.5.*`, rewrite to `>=3.5`.
let patched = match s {
">=3.12.*" => Some(">=3.12"),
">=3.11.*" => Some(">=3.11"),
">=3.10.*" => Some(">=3.10"),
">=3.9.*" => Some(">=3.9"),
">=3.8.*" => Some(">=3.8"),
">=3.7.*" => Some(">=3.7"),
">=3.6.*" => Some(">=3.6"),
">=3.5.*" => Some(">=3.5"),
">=3.4.*" => Some(">=3.4"),
">=3.3.*" => Some(">=3.3"),
">=3.2.*" => Some(">=3.2"),
">=3.1.*" => Some(">=3.1"),
">=3.0.*" => Some(">=3.0"),
_ => None,
};
if let Some(patched) = patched {
if let Ok(specifier) = VersionSpecifiers::from_str(patched) {
warn!(
"Correcting invalid wildcard bound on version specifier (before: `{s}`; after: `{patched}`)",
);
return Ok(Self(specifier));
}
}
Err(err)
}
}
}
}
impl From<LenientVersionSpecifiers> for VersionSpecifiers {
fn from(specifiers: LenientVersionSpecifiers) -> Self {
specifiers.0
}
}
impl<'de> Deserialize<'de> for LenientVersionSpecifiers {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(de::Error::custom)
}
}

2
crates/pypi-types/src/lib.rs
View file

@ -1,7 +1,9 @@
pub use direct_url::{ArchiveInfo, DirectUrl, VcsInfo, VcsKind};
pub use lenient_requirement::LenientVersionSpecifiers;
pub use metadata::{Error, Metadata21};
pub use simple_json::{File, SimpleJson};
mod direct_url;
mod lenient_requirement;
mod metadata;
mod simple_json;

49
crates/pypi-types/src/metadata.rs
View file

@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::warn;
use crate::lenient_requirement::LenientVersionSpecifiers;
use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
use pep508_rs::{Pep508Error, Requirement};
use puffin_normalize::{ExtraName, InvalidNameError, PackageName};
@ -271,54 +272,6 @@ impl From<LenientRequirement> for Requirement {
}
}
/// Like [`VersionSpecifiers`], but attempts to correct some common errors in user-provided requirements.
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
struct LenientVersionSpecifiers(VersionSpecifiers);
impl FromStr for LenientVersionSpecifiers {
type Err = Pep440Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match VersionSpecifiers::from_str(s) {
Ok(specifiers) => Ok(Self(specifiers)),
Err(err) => {
// Given `>=3.5.*`, rewrite to `>=3.5`.
let patched = match s {
">=3.12.*" => Some(">=3.12"),
">=3.11.*" => Some(">=3.11"),
">=3.10.*" => Some(">=3.10"),
">=3.9.*" => Some(">=3.9"),
">=3.8.*" => Some(">=3.8"),
">=3.7.*" => Some(">=3.7"),
">=3.6.*" => Some(">=3.6"),
">=3.5.*" => Some(">=3.5"),
">=3.4.*" => Some(">=3.4"),
">=3.3.*" => Some(">=3.3"),
">=3.2.*" => Some(">=3.2"),
">=3.1.*" => Some(">=3.1"),
">=3.0.*" => Some(">=3.0"),
_ => None,
};
if let Some(patched) = patched {
if let Ok(specifier) = VersionSpecifiers::from_str(patched) {
warn!(
"Correcting invalid wildcard bound on version specifier (before: `{s}`; after: `{patched}`)",
);
return Ok(Self(specifier));
}
}
Err(err)
}
}
}
}
impl From<LenientVersionSpecifiers> for VersionSpecifiers {
fn from(specifiers: LenientVersionSpecifiers) -> Self {
specifiers.0
}
}
#[cfg(test)]
mod tests {
use std::str::FromStr;

25
crates/pypi-types/src/simple_json.rs
View file

@ -1,4 +1,8 @@
use serde::{Deserialize, Serialize};
use pep440_rs::VersionSpecifiers;
use serde::{de, Deserialize, Deserializer, Serialize};
use std::str::FromStr;
use crate::lenient_requirement::LenientVersionSpecifiers;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SimpleJson {
@ -15,13 +19,30 @@ pub struct File {
pub data_dist_info_metadata: Metadata,
pub filename: String,
pub hashes: Hashes,
pub requires_python: Option<String>,
/// Note: Deserialized with [`LenientVersionSpecifiers`] since there are a number of invalid
/// versions on pypi
#[serde(deserialize_with = "deserialize_version_specifiers_lenient")]
pub requires_python: Option<VersionSpecifiers>,
pub size: usize,
pub upload_time: String,
pub url: String,
pub yanked: Yanked,
}
fn deserialize_version_specifiers_lenient<'de, D>(
deserializer: D,
) -> Result<Option<VersionSpecifiers>, D::Error>
where
D: Deserializer<'de>,
{
let maybe_string: Option<String> = Option::deserialize(deserializer)?;
let Some(string) = maybe_string else {
return Ok(None);
};
let lenient = LenientVersionSpecifiers::from_str(&string).map_err(de::Error::custom)?;
Ok(Some(lenient.into()))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Metadata {