Filter out incompatible dists (#398)

Filter out source dists and wheels whose `requires-python` from the
simple api is incompatible with the current python version.

This change showed an important problem: When we use a fake python
version for resolving, building source distributions breaks down because
we can only build with versions we actually have.

This change became surprisingly big. The tests now require python 3.7 to
be installed, but changing that would mean an even bigger change.

Fixes #388

crates/pypi-types/src/lenient_requirement.rs

@@ -0,0 +1,64 @@
+use pep440_rs::{Pep440Error, VersionSpecifiers};
+use serde::{de, Deserialize, Deserializer, Serialize};
+use std::str::FromStr;
+use tracing::warn;
+
+/// Like [`VersionSpecifiers`], but attempts to correct some common errors in user-provided requirements.
+///
+/// We turn `>=3.x.*` into `>=3.x`
+#[derive(Debug, Clone, Serialize, Eq, PartialEq)]
+pub struct LenientVersionSpecifiers(VersionSpecifiers);
+
+impl FromStr for LenientVersionSpecifiers {
+    type Err = Pep440Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match VersionSpecifiers::from_str(s) {
+            Ok(specifiers) => Ok(Self(specifiers)),
+            Err(err) => {
+                // Given `>=3.5.*`, rewrite to `>=3.5`.
+                let patched = match s {
+                    ">=3.12.*" => Some(">=3.12"),
+                    ">=3.11.*" => Some(">=3.11"),
+                    ">=3.10.*" => Some(">=3.10"),
+                    ">=3.9.*" => Some(">=3.9"),
+                    ">=3.8.*" => Some(">=3.8"),
+                    ">=3.7.*" => Some(">=3.7"),
+                    ">=3.6.*" => Some(">=3.6"),
+                    ">=3.5.*" => Some(">=3.5"),
+                    ">=3.4.*" => Some(">=3.4"),
+                    ">=3.3.*" => Some(">=3.3"),
+                    ">=3.2.*" => Some(">=3.2"),
+                    ">=3.1.*" => Some(">=3.1"),
+                    ">=3.0.*" => Some(">=3.0"),
+                    _ => None,
+                };
+                if let Some(patched) = patched {
+                    if let Ok(specifier) = VersionSpecifiers::from_str(patched) {
+                        warn!(
+                        "Correcting invalid wildcard bound on version specifier (before: `{s}`; after: `{patched}`)",
+                    );
+                        return Ok(Self(specifier));
+                    }
+                }
+                Err(err)
+            }
+        }
+    }
+}
+
+impl From<LenientVersionSpecifiers> for VersionSpecifiers {
+    fn from(specifiers: LenientVersionSpecifiers) -> Self {
+        specifiers.0
+    }
+}
+
+impl<'de> Deserialize<'de> for LenientVersionSpecifiers {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        Self::from_str(&s).map_err(de::Error::custom)
+    }
+}

crates/pypi-types/src/lib.rs

@@ -1,7 +1,9 @@
 pub use direct_url::{ArchiveInfo, DirectUrl, VcsInfo, VcsKind};
+pub use lenient_requirement::LenientVersionSpecifiers;
 pub use metadata::{Error, Metadata21};
 pub use simple_json::{File, SimpleJson};
 
 mod direct_url;
+mod lenient_requirement;
 mod metadata;
 mod simple_json;

crates/pypi-types/src/metadata.rs

@@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tracing::warn;
 
+use crate::lenient_requirement::LenientVersionSpecifiers;
 use pep440_rs::{Pep440Error, Version, VersionSpecifiers};
 use pep508_rs::{Pep508Error, Requirement};
 use puffin_normalize::{ExtraName, InvalidNameError, PackageName};
@@ -271,54 +272,6 @@ impl From<LenientRequirement> for Requirement {
     }
 }
 
-/// Like [`VersionSpecifiers`], but attempts to correct some common errors in user-provided requirements.
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
-struct LenientVersionSpecifiers(VersionSpecifiers);
-
-impl FromStr for LenientVersionSpecifiers {
-    type Err = Pep440Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match VersionSpecifiers::from_str(s) {
-            Ok(specifiers) => Ok(Self(specifiers)),
-            Err(err) => {
-                // Given `>=3.5.*`, rewrite to `>=3.5`.
-                let patched = match s {
-                    ">=3.12.*" => Some(">=3.12"),
-                    ">=3.11.*" => Some(">=3.11"),
-                    ">=3.10.*" => Some(">=3.10"),
-                    ">=3.9.*" => Some(">=3.9"),
-                    ">=3.8.*" => Some(">=3.8"),
-                    ">=3.7.*" => Some(">=3.7"),
-                    ">=3.6.*" => Some(">=3.6"),
-                    ">=3.5.*" => Some(">=3.5"),
-                    ">=3.4.*" => Some(">=3.4"),
-                    ">=3.3.*" => Some(">=3.3"),
-                    ">=3.2.*" => Some(">=3.2"),
-                    ">=3.1.*" => Some(">=3.1"),
-                    ">=3.0.*" => Some(">=3.0"),
-                    _ => None,
-                };
-                if let Some(patched) = patched {
-                    if let Ok(specifier) = VersionSpecifiers::from_str(patched) {
-                        warn!(
-                        "Correcting invalid wildcard bound on version specifier (before: `{s}`; after: `{patched}`)",
-                    );
-                        return Ok(Self(specifier));
-                    }
-                }
-                Err(err)
-            }
-        }
-    }
-}
-
-impl From<LenientVersionSpecifiers> for VersionSpecifiers {
-    fn from(specifiers: LenientVersionSpecifiers) -> Self {
-        specifiers.0
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::str::FromStr;

crates/pypi-types/src/simple_json.rs

@@ -1,4 +1,8 @@
-use serde::{Deserialize, Serialize};
+use pep440_rs::VersionSpecifiers;
+use serde::{de, Deserialize, Deserializer, Serialize};
+use std::str::FromStr;
+
+use crate::lenient_requirement::LenientVersionSpecifiers;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SimpleJson {
@@ -15,13 +19,30 @@ pub struct File {
     pub data_dist_info_metadata: Metadata,
     pub filename: String,
     pub hashes: Hashes,
-    pub requires_python: Option<String>,
+    /// Note: Deserialized with [`LenientVersionSpecifiers`] since there are a number of invalid
+    /// versions on pypi
+    #[serde(deserialize_with = "deserialize_version_specifiers_lenient")]
+    pub requires_python: Option<VersionSpecifiers>,
     pub size: usize,
     pub upload_time: String,
     pub url: String,
     pub yanked: Yanked,
 }
 
+fn deserialize_version_specifiers_lenient<'de, D>(
+    deserializer: D,
+) -> Result<Option<VersionSpecifiers>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let maybe_string: Option<String> = Option::deserialize(deserializer)?;
+    let Some(string) = maybe_string else {
+        return Ok(None);
+    };
+    let lenient = LenientVersionSpecifiers::from_str(&string).map_err(de::Error::custom)?;
+    Ok(Some(lenient.into()))
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum Metadata {