Add an additional requirements fixup (#174)

Also checking in a variety of different requirements inputs.

crates/puffin-package/src/metadata.rs

@@ -199,7 +199,9 @@ impl Metadata21 {
     }
 }
 
-static REQUIREMENT_FIXUP_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d)([<>=~^!])").unwrap());
+static MISSING_COMMA: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d)([<>=~^!])").unwrap());
+
+static NOT_EQUAL_TILDE: Lazy<Regex> = Lazy::new(|| Regex::new(r"!=~((?:\d\.)*\d)").unwrap());
 
 /// Like [`Requirement`], but attempts to correct some common errors in user-provided requirements.
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -213,7 +215,7 @@ impl FromStr for LenientRequirement {
             Ok(requirement) => Ok(Self(requirement)),
             Err(err) => {
                 // Given `elasticsearch-dsl (>=7.2.0<8.0.0)`, rewrite to `elasticsearch-dsl (>=7.2.0,<8.0.0)`.
-                let patched = REQUIREMENT_FIXUP_REGEX.replace(s, r"$1,$2");
+                let patched = MISSING_COMMA.replace(s, r"$1,$2");
                 if patched != s {
                     if let Ok(requirement) = Requirement::from_str(&patched) {
                         warn!(
@@ -222,6 +224,18 @@ impl FromStr for LenientRequirement {
                         return Ok(Self(requirement));
                     }
                 }
+
+                // Given `jupyter-core (!=~5.0,>=4.12)`, rewrite to `jupyter-core (!=5.0.*,>=4.12)`.
+                let patched = NOT_EQUAL_TILDE.replace(s, r"!=${1}.*");
+                if patched != s {
+                    if let Ok(requirement) = Requirement::from_str(&patched) {
+                        warn!(
+                        "Adding wildcard after invalid tilde operator (before: `{s}`; after: `{patched}`)",
+                    );
+                        return Ok(Self(requirement));
+                    }
+                }
+
                 Err(err)
             }
         }
@@ -281,3 +295,37 @@ impl From<LenientVersionSpecifiers> for VersionSpecifiers {
         specifiers.0
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+
+    use pep508_rs::Requirement;
+
+    use crate::metadata::LenientRequirement;
+
+    #[test]
+    fn missing_comma() {
+        let actual: Requirement = LenientRequirement::from_str("elasticsearch-dsl (>=7.2.0<8.0.0)")
+            .unwrap()
+            .into();
+        let expected: Requirement =
+            Requirement::from_str("elasticsearch-dsl (>=7.2.0,<8.0.0)").unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn not_equal_tile() {
+        let actual: Requirement = LenientRequirement::from_str("jupyter-core (!=~5.0,>=4.12)")
+            .unwrap()
+            .into();
+        let expected: Requirement = Requirement::from_str("jupyter-core (!=5.0.*,>=4.12)").unwrap();
+        assert_eq!(actual, expected);
+
+        let actual: Requirement = LenientRequirement::from_str("jupyter-core (!=~5,>=4.12)")
+            .unwrap()
+            .into();
+        let expected: Requirement = Requirement::from_str("jupyter-core (!=5.*,>=4.12)").unwrap();
+        assert_eq!(actual, expected);
+    }
+}

scripts/benchmarks/requirements.in

@@ -6,3 +6,25 @@ pygls>=1.0.1
 lsprotocol>=2023.0.0a1
 ruff>=0.0.274
 typing_extensions
+scipy
+numpy
+pandas<2.0.0
+matplotlib>=3.0.0
+scikit-learn
+rich
+textual
+jupyter>=1.0.0,<2.0.0
+transformers[torch]
+django<4.0.0
+sqlalchemy
+psycopg2-binary
+trio<0.20
+trio-websocket
+trio-asyncio
+trio-typing
+trio-protocol
+fastapi
+typer
+pydantic
+uvicorn
+traitlets

scripts/benchmarks/requirements/flyte.in

@@ -0,0 +1,35 @@
+# Source: https://github.com/flyteorg/flytekit/blob/2a5fa355fdd790f525731b3634a96579e9f744e1/dev-requirements.in
+coverage[toml]
+hypothesis
+joblib
+mock
+pytest
+pytest-asyncio
+pytest-cov
+mypy
+pre-commit
+codespell
+google-cloud-bigquery
+google-cloud-bigquery-storage
+IPython
+keyrings.alt
+
+# Only install tensorflow if not running on an arm Mac.
+tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
+# Tensorflow release candidate supports python 3.11
+tensorflow==2.13.0; python_version>='3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
+
+# Newer versions of torch bring in nvidia dependencies that are not present in windows, so
+# we put this constraint while we do not have per-environment requirements files
+torch<=1.12.1; python_version<'3.11'
+# pytorch 2 supports python 3.11
+torch<=2.0.0; python_version>='3.11' or platform_system!='Windows'
+
+pillow
+scikit-learn
+types-protobuf
+types-croniter
+types-mock
+autoflake
+types-requests
+prometheus-client

scripts/benchmarks/requirements/scispacy.in

@@ -0,0 +1,29 @@
+# Source: https://github.com/allenai/scispacy/blob/0969c54d21e18a171fc90f1478392412e2c54894/requirements.in
+numpy
+scipy<1.11
+spacy>=3.6.0,<3.7.0
+spacy-lookups-data
+pandas
+requests>=2.0.0,<3.0.0
+conllu
+
+# Candidate generation and entity linking
+joblib
+scikit-learn>=0.20.3
+
+# Required for testing.
+pytest
+pytest-cov
+flake8
+# black currently pinned because of a dependency issue with spacy, typer, and click
+black
+mypy
+types-requests
+types-setuptools
+types-tabulate
+
+# Required for releases.
+twine
+
+# required for the tests to run, or to use the custom sentence splitter
+pysbd

scripts/benchmarks/requirements/slow.in

@@ -0,0 +1,16 @@
+# Source: https://github.com/jazzband/pip-tools/issues/1829#issue-1642121710
+bert_score
+blobfile
+nltk
+numpy<1.24.0
+packaging
+psutil
+PyYAML
+setuptools
+spacy
+torch
+torchmetrics
+tqdm
+transformers
+wandb
+datasets

scripts/benchmarks/requirements/trio.in

@@ -0,0 +1,23 @@
+# Source: https://github.com/python-trio/trio/blob/01638f87a7344031c10b95a76546d21ccc888c5d/docs-requirements.in
+# RTD is currently installing 1.5.3, which has a bug in :lineno-match:
+sphinx >= 4.0, < 6.2
+jinja2
+sphinx_rtd_theme
+sphinxcontrib-jquery
+sphinxcontrib-trio
+towncrier
+
+# Trio's own dependencies
+cffi; os_name == "nt"
+attrs >= 19.2.0
+sortedcontainers
+idna
+outcome
+sniffio
+exceptiongroup >= 1.0.0rc9
+
+# See note in test-requirements.in
+immutables >= 0.6
+
+# types used in annotations
+pyOpenSSL