From 496cb7b2ef01ff9d39fffa4bb3257ffa814d89fe Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 12 Oct 2023 13:09:00 -0400 Subject: [PATCH] Migrate to `requirements_txt.rs` (#90) Remove the parser I wrote in favor of Konsti's which is much more complete. The only change vs. the version in `poc-monotrail` is that I changed the tests to use insta rather than manually storing and comparing against JSON snapshots. Closes https://github.com/astral-sh/puffin/issues/89. --- Cargo.lock | 249 +++----- Cargo.toml | 1 + crates/puffin-cli/src/commands/compile.rs | 14 +- crates/puffin-cli/src/commands/sync.rs | 13 +- crates/puffin-package/Cargo.toml | 12 +- crates/puffin-package/benches/parser.rs | 96 --- crates/puffin-package/src/lib.rs | 2 +- crates/puffin-package/src/requirements.rs | 324 ---------- crates/puffin-package/src/requirements_txt.rs | 582 ++++++++++++++++++ ...age__requirements__tests__pip_compile.snap | 320 ---------- ..._package__requirements__tests__simple.snap | 35 -- ...nts_txt__test__line-endings-basic.txt.snap | 199 ++++++ ..._test__line-endings-constraints-a.txt.snap | 96 +++ ..._test__line-endings-constraints-b.txt.snap | 72 +++ ...nts_txt__test__line-endings-empty.txt.snap | 8 + ...xt__test__line-endings-for-poetry.txt.snap | 129 ++++ ...txt__test__line-endings-include-a.txt.snap | 51 ++ ...txt__test__line-endings-include-b.txt.snap | 19 + ...__line-endings-poetry-with-hashes.txt.snap | 329 ++++++++++ ...nts_txt__test__line-endings-small.txt.snap | 73 +++ ...xt__test__line-endings-whitespace.txt.snap | 51 ++ ...quirements_txt__test__parse-basic.txt.snap | 199 ++++++ ...ts_txt__test__parse-constraints-a.txt.snap | 96 +++ ...ts_txt__test__parse-constraints-b.txt.snap | 72 +++ ...quirements_txt__test__parse-empty.txt.snap | 8 + ...ments_txt__test__parse-for-poetry.txt.snap | 129 ++++ ...ements_txt__test__parse-include-a.txt.snap | 51 ++ ...ements_txt__test__parse-include-b.txt.snap | 19 + ...t__test__parse-poetry-with-hashes.txt.snap | 329 ++++++++++ ...quirements_txt__test__parse-small.txt.snap | 73 +++ ...ments_txt__test__parse-whitespace.txt.snap | 51 ++ .../test-data/requirements-txt/basic.txt | 6 + .../requirements-txt/constraints-a.txt | 2 + .../requirements-txt/constraints-b.txt | 2 + .../test-data/requirements-txt/empty.txt | 0 .../test-data/requirements-txt/for-poetry.txt | 6 + .../test-data/requirements-txt/include-a.txt | 3 + .../test-data/requirements-txt/include-b.txt | 1 + .../requirements-txt/invalid-include | 1 + .../requirements-txt/invalid-requirement | 1 + .../requirements-txt/poetry-with-hashes.txt | 14 + .../test-data/requirements-txt/small.txt | 4 + .../test-data/requirements-txt/whitespace.txt | 25 + 43 files changed, 2817 insertions(+), 950 deletions(-) delete mode 100644 crates/puffin-package/benches/parser.rs delete mode 100644 crates/puffin-package/src/requirements.rs create mode 100644 crates/puffin-package/src/requirements_txt.rs delete mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements__tests__pip_compile.snap delete mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements__tests__simple.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-basic.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-a.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-b.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-empty.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-for-poetry.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-a.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-b.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-poetry-with-hashes.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-small.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-whitespace.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-basic.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-a.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-b.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-empty.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-for-poetry.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-a.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-b.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-poetry-with-hashes.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-small.txt.snap create mode 100644 crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-whitespace.txt.snap create mode 100644 crates/puffin-package/test-data/requirements-txt/basic.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/constraints-a.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/constraints-b.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/empty.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/for-poetry.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/include-a.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/include-b.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/invalid-include create mode 100644 crates/puffin-package/test-data/requirements-txt/invalid-requirement create mode 100644 crates/puffin-package/test-data/requirements-txt/poetry-with-hashes.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/small.txt create mode 100644 crates/puffin-package/test-data/requirements-txt/whitespace.txt diff --git a/Cargo.lock b/Cargo.lock index f9b910576..54d8cc4b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -67,12 +67,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "0.6.4" @@ -317,12 +311,6 @@ dependencies = [ "serde", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.0.83" @@ -370,33 +358,6 @@ dependencies = [ "stacker", ] -[[package]] -name = "ciborium" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" - -[[package]] -name = "ciborium-ll" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "clap" version = "4.4.6" @@ -516,42 +477,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1007,12 +932,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - [[package]] name = "hashbrown" version = "0.12.3" @@ -1355,15 +1274,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.11.0" @@ -1417,9 +1327,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45786cec4d5e54a224b15cb9f06751883103a27c19c93eda09b0b4f5f08fefac" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" @@ -1608,12 +1518,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" - [[package]] name = "openssl" version = "0.10.57" @@ -1833,34 +1737,6 @@ dependencies = [ "time", ] -[[package]] -name = "plotters" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" - -[[package]] -name = "plotters-svg" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.4.3" @@ -1874,10 +1750,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] -name = "proc-macro2" -version = "1.0.68" +name = "proc-macro-error" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -1905,7 +1805,7 @@ dependencies = [ "gourgeist", "indicatif", "install-wheel-rs", - "itertools 0.11.0", + "itertools", "pep440_rs", "pep508_rs", "platform-host", @@ -1982,7 +1882,8 @@ name = "puffin-package" version = "0.0.1" dependencies = [ "anyhow", - "criterion", + "fs-err", + "indoc 2.0.4", "insta", "mailparse", "memchr", @@ -1992,7 +1893,12 @@ dependencies = [ "regex", "rfc2047-decoder", "serde", + "serde_json", + "tempfile", + "test-case", "thiserror", + "tracing", + "unscanny", ] [[package]] @@ -2244,14 +2150,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.1", + "regex-syntax 0.8.1", ] [[package]] @@ -2265,13 +2171,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.1", ] [[package]] @@ -2282,9 +2188,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" [[package]] name = "reqwest" @@ -2374,9 +2280,9 @@ checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" [[package]] name = "retry-policies" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a715dc4d0e8aea3085a9a94d76e79c79c7df7c9f6be609da841a6d2489ca3687" +checksum = "17dd00bff1d737c40dbcd47d4375281bf4c17933f9eef0a185fc7bacca23ecbd" dependencies = [ "anyhow", "chrono", @@ -2405,9 +2311,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.17" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ "bitflags 2.4.0", "errno", @@ -2745,6 +2651,41 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "test-case" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8f1e820b7f1d95a0cdbf97a5df9de10e1be731983ab943e56703ac1b8e9d425" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" +dependencies = [ + "cfg-if", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "test-case-macros" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cfd7bbc88a0104e304229fba519bdc45501a30b760fb72240342f1289ad257" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", + "test-case-core", +] + [[package]] name = "testing_logger" version = "0.1.1" @@ -2812,16 +2753,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -3036,6 +2967,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +[[package]] +name = "unscanny" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47" + [[package]] name = "url" version = "2.4.1" diff --git a/Cargo.toml b/Cargo.toml index acd4da17d..b70dd9cc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ tracing = { version = "0.1.37" } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } tracing-tree = { version = "0.2.5" } unicode-width = { version = "0.1.8" } +unscanny = { version = "0.1.0" } url = { version = "2.4.1" } walkdir = { version = "2.4.0" } which = { version = "4.4.2" } diff --git a/crates/puffin-cli/src/commands/compile.rs b/crates/puffin-cli/src/commands/compile.rs index 7d9f1dc69..ebd81b3a6 100644 --- a/crates/puffin-cli/src/commands/compile.rs +++ b/crates/puffin-cli/src/commands/compile.rs @@ -1,6 +1,5 @@ use std::fmt::Write; use std::path::Path; -use std::str::FromStr; use anyhow::Result; use colored::Colorize; @@ -10,7 +9,7 @@ use platform_host::Platform; use platform_tags::Tags; use puffin_client::PypiClientBuilder; use puffin_interpreter::PythonExecutable; -use puffin_package::requirements::Requirements; +use puffin_package::requirements_txt::RequirementsTxt; use crate::commands::reporters::ResolverReporter; use crate::commands::{elapsed, ExitStatus}; @@ -25,11 +24,12 @@ pub(crate) async fn compile( let start = std::time::Instant::now(); // Read the `requirements.txt` from disk. - let requirements_txt = std::fs::read_to_string(src)?; - - // Parse the `requirements.txt` into a list of requirements. - let requirements = Requirements::from_str(&requirements_txt)?; - + let requirements_txt = RequirementsTxt::parse(src, std::env::current_dir()?)?; + let requirements = requirements_txt + .requirements + .into_iter() + .map(|entry| entry.requirement) + .collect::>(); if requirements.is_empty() { writeln!(printer, "No requirements found")?; return Ok(ExitStatus::Success); diff --git a/crates/puffin-cli/src/commands/sync.rs b/crates/puffin-cli/src/commands/sync.rs index 8796e8b53..51dc9f2de 100644 --- a/crates/puffin-cli/src/commands/sync.rs +++ b/crates/puffin-cli/src/commands/sync.rs @@ -1,6 +1,5 @@ use std::fmt::Write; use std::path::Path; -use std::str::FromStr; use anyhow::Result; use bitflags::bitflags; @@ -14,7 +13,7 @@ use puffin_client::PypiClientBuilder; use puffin_installer::{LocalIndex, RemoteDistribution}; use puffin_interpreter::{PythonExecutable, SitePackages}; use puffin_package::package_name::PackageName; -use puffin_package::requirements::Requirements; +use puffin_package::requirements_txt::RequirementsTxt; use crate::commands::reporters::{ DownloadReporter, InstallReporter, ResolverReporter, UnzipReporter, @@ -40,10 +39,12 @@ pub(crate) async fn sync( let start = std::time::Instant::now(); // Read the `requirements.txt` from disk. - let requirements_txt = std::fs::read_to_string(src)?; - - // Parse the `requirements.txt` into a list of requirements. - let requirements = Requirements::from_str(&requirements_txt)?; + let requirements_txt = RequirementsTxt::parse(src, std::env::current_dir()?)?; + let requirements = requirements_txt + .requirements + .into_iter() + .map(|entry| entry.requirement) + .collect::>(); if requirements.is_empty() { writeln!(printer, "No requirements found")?; return Ok(ExitStatus::Success); diff --git a/crates/puffin-package/Cargo.toml b/crates/puffin-package/Cargo.toml index 0d53b5d31..4b796bb59 100644 --- a/crates/puffin-package/Cargo.toml +++ b/crates/puffin-package/Cargo.toml @@ -8,6 +8,7 @@ pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] } anyhow = { workspace = true } +fs-err = { workspace = true } mailparse = { workspace = true } memchr = { workspace = true } once_cell = { workspace = true } @@ -15,11 +16,12 @@ regex = { workspace = true } rfc2047-decoder = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +tracing.workspace = true +unscanny = { workspace = true } [dev-dependencies] -criterion = { version = "0.5.1" } +indoc = { version = "2.0.4" } insta = { version = "1.33.0" } - -[[bench]] -name = "parser" -harness = false +serde_json = { version = "1.0.107" } +tempfile = { version = "3.8.0" } +test-case = { version = "3.2.1" } diff --git a/crates/puffin-package/benches/parser.rs b/crates/puffin-package/benches/parser.rs deleted file mode 100644 index fcecec6e0..000000000 --- a/crates/puffin-package/benches/parser.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::str::FromStr; - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; - -use puffin_package::requirements::Requirements; - -const REQUIREMENTS_TXT: &str = r" -# -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements.txt --resolver=backtracking pyproject.toml -# -attrs==23.1.0 \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 - # via - # cattrs - # lsprotocol -cattrs==23.1.2 \ - --hash=sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4 \ - --hash=sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657 - # via lsprotocol -exceptiongroup==1.1.3 \ - --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \ - --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3 - # via cattrs -importlib-metadata==6.7.0 \ - --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \ - --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5 - # via - # attrs - # typeguard -lsprotocol==2023.0.0b1 \ - --hash=sha256:ade2cd0fa0ede7965698cb59cd05d3adbd19178fd73e83f72ef57a032fbb9d62 \ - --hash=sha256:f7a2d4655cbd5639f373ddd1789807450c543341fa0a32b064ad30dbb9f510d4 - # via - # pygls - # ruff-lsp (pyproject.toml) -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 - # via ruff-lsp (pyproject.toml) -pygls==1.1.0 \ - --hash=sha256:70acb6fe0df1c8a17b7ce08daa0afdb4aedc6913a6a6696003e1434fda80a06e \ - --hash=sha256:eb19b818039d3d705ec8adbcdf5809a93af925f30cd7a3f3b7573479079ba00e - # via ruff-lsp (pyproject.toml) -ruff==0.0.292 \ - --hash=sha256:02f29db018c9d474270c704e6c6b13b18ed0ecac82761e4fcf0faa3728430c96 \ - --hash=sha256:1093449e37dd1e9b813798f6ad70932b57cf614e5c2b5c51005bf67d55db33ac \ - --hash=sha256:69654e564342f507edfa09ee6897883ca76e331d4bbc3676d8a8403838e9fade \ - --hash=sha256:6bdfabd4334684a4418b99b3118793f2c13bb67bf1540a769d7816410402a205 \ - --hash=sha256:6c3c91859a9b845c33778f11902e7b26440d64b9d5110edd4e4fa1726c41e0a4 \ - --hash=sha256:7f67a69c8f12fbc8daf6ae6d36705037bde315abf8b82b6e1f4c9e74eb750f68 \ - --hash=sha256:87616771e72820800b8faea82edd858324b29bb99a920d6aa3d3949dd3f88fb0 \ - --hash=sha256:8e087b24d0d849c5c81516ec740bf4fd48bf363cfb104545464e0fca749b6af9 \ - --hash=sha256:9889bac18a0c07018aac75ef6c1e6511d8411724d67cb879103b01758e110a81 \ - --hash=sha256:aa7c77c53bfcd75dbcd4d1f42d6cabf2485d2e1ee0678da850f08e1ab13081a8 \ - --hash=sha256:ac153eee6dd4444501c4bb92bff866491d4bfb01ce26dd2fff7ca472c8df9ad0 \ - --hash=sha256:b76deb3bdbea2ef97db286cf953488745dd6424c122d275f05836c53f62d4016 \ - --hash=sha256:be8eb50eaf8648070b8e58ece8e69c9322d34afe367eec4210fdee9a555e4ca7 \ - --hash=sha256:e854b05408f7a8033a027e4b1c7f9889563dd2aca545d13d06711e5c39c3d003 \ - --hash=sha256:f160b5ec26be32362d0774964e218f3fcf0a7da299f7e220ef45ae9e3e67101a \ - --hash=sha256:f27282bedfd04d4c3492e5c3398360c9d86a295be00eccc63914438b4ac8a83c \ - --hash=sha256:f4476f1243af2d8c29da5f235c13dca52177117935e1f9393f9d90f9833f69e4 - # via ruff-lsp (pyproject.toml) -typeguard==3.0.2 \ - --hash=sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e \ - --hash=sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a - # via pygls -typing-extensions==4.7.1 \ - --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ - --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 - # via - # cattrs - # importlib-metadata - # ruff-lsp (pyproject.toml) - # typeguard -zipp==3.15.0 \ - --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ - --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 - # via importlib-metadata -"; - -fn bench_fibs(c: &mut Criterion) { - let mut group = c.benchmark_group("Parser"); - - group.bench_function("Parse", |b| { - b.iter(|| Requirements::from_str(black_box(REQUIREMENTS_TXT)).unwrap()); - }); - - group.finish(); -} - -criterion_group!(benches, bench_fibs); -criterion_main!(benches); diff --git a/crates/puffin-package/src/lib.rs b/crates/puffin-package/src/lib.rs index f35d5d5e6..c0f3bd132 100644 --- a/crates/puffin-package/src/lib.rs +++ b/crates/puffin-package/src/lib.rs @@ -1,4 +1,4 @@ pub mod dist_info_name; pub mod metadata; pub mod package_name; -pub mod requirements; +pub mod requirements_txt; diff --git a/crates/puffin-package/src/requirements.rs b/crates/puffin-package/src/requirements.rs deleted file mode 100644 index f3a2ca8c1..000000000 --- a/crates/puffin-package/src/requirements.rs +++ /dev/null @@ -1,324 +0,0 @@ -use std::borrow::Cow; -use std::str::FromStr; - -use anyhow::Result; -use memchr::{memchr2, memchr_iter}; - -use pep508_rs::{Pep508Error, Requirement}; - -#[derive(Debug)] -pub struct Requirements(Vec); - -impl Requirements { - pub fn new(requirements: Vec) -> Self { - Self(requirements) - } - - /// Filter the requirements. - #[must_use] - pub fn filter(self, mut f: F) -> Self - where - F: FnMut(&Requirement) -> bool, - { - Self( - self.0 - .into_iter() - .filter(|requirement| f(requirement)) - .collect(), - ) - } - - /// Return the number of requirements. - pub fn len(&self) -> usize { - self.0.len() - } - - /// Return `true` if there are no requirements. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Return an iterator over the requirements. - pub fn iter(&self) -> impl Iterator { - self.0.iter() - } -} - -impl IntoIterator for Requirements { - type Item = Requirement; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl FromStr for Requirements { - type Err = Pep508Error; - - fn from_str(s: &str) -> Result { - Ok(Self::new( - RequirementsIterator::new(s) - .map(|requirement| Requirement::from_str(requirement.as_str())) - .collect::, Pep508Error>>()?, - )) - } -} - -#[derive(Debug)] -struct RequirementsIterator<'a> { - text: &'a str, - index: usize, -} - -impl<'a> RequirementsIterator<'a> { - fn new(text: &'a str) -> Self { - Self { text, index: 0 } - } -} - -#[derive(Debug)] -struct RequirementLine<'a> { - /// The line as included in the `requirements.txt`, including comments and `--hash` extras. - line: Cow<'a, str>, - /// The line, with comments and `--hash` extras stripped. - len: usize, -} - -impl<'a> RequirementLine<'a> { - /// Create a new `RequirementLine` from a line of text. - fn from_line(line: Cow<'a, str>) -> Self { - Self { - len: Self::strip_trivia(&line), - line, - } - } - - /// Return a parseable requirement line. - fn as_str(&self) -> &str { - &self.line[..self.len] - } - - /// Strip trivia (comments and `--hash` extras) from a requirement, returning the length of the - /// requirement itself. - fn strip_trivia(requirement: &str) -> usize { - let mut len = requirement.len(); - - // Strip comments. - for position in memchr_iter(b'#', requirement[..len].as_bytes()) { - // The comment _must_ be preceded by whitespace. - if requirement[..len + position] - .chars() - .next_back() - .is_some_and(char::is_whitespace) - { - len = position; - break; - } - } - - // Strip `--hash` extras. - if let Some(index) = requirement[..len].find("--hash") { - len = index; - } - - len - } -} - -impl<'a> Iterator for RequirementsIterator<'a> { - type Item = RequirementLine<'a>; - - #[inline] - fn next(&mut self) -> Option> { - if self.index == self.text.len() { - return None; - } - - // Find the next line break. - let Some((start, length)) = find_newline(&self.text[self.index..]) else { - // Parse the rest of the text. - let line = &self.text[self.index..]; - self.index = self.text.len(); - - // Skip fully-commented lines. - if line.trim_start().starts_with('#') { - return None; - } - - // Skip empty lines. - if line.trim().is_empty() { - return None; - } - - return Some(RequirementLine::from_line(Cow::Borrowed(line))); - }; - - // Skip fully-commented lines. - if self.text[self.index..].trim_start().starts_with('#') { - self.index += start + length; - return self.next(); - } - - // Skip empty lines. - if self.text[self.index..self.index + start].trim().is_empty() { - self.index += start + length; - return self.next(); - } - - // If the newline is preceded by a continuation (\\), keep going. - if self.text[..self.index + start] - .chars() - .next_back() - .is_some_and(|c| c == '\\') - { - // Add the line contents, preceding the continuation. - let mut line = self.text[self.index..self.index + start - 1].to_owned(); - self.index += start + length; - - // Eat lines until we see a non-continuation. - while let Some((start, length)) = find_newline(&self.text[self.index..]) { - if self.text[..self.index + start] - .chars() - .next_back() - .is_some_and(|c| c == '\\') - { - // Add the line contents, preceding the continuation. - line.push_str(&self.text[self.index..self.index + start - 1]); - self.index += start + length; - } else { - // Add the line contents, excluding the continuation. - line.push_str(&self.text[self.index..self.index + start]); - self.index += start + length; - break; - } - } - - Some(RequirementLine::from_line(Cow::Owned(line))) - } else { - let line = &self.text[self.index..self.index + start]; - self.index += start + length; - Some(RequirementLine::from_line(Cow::Borrowed(line))) - } - } -} - -/// Return the start and end position of the first newline character in the given text. -#[inline] -fn find_newline(text: &str) -> Option<(usize, usize)> { - let bytes = text.as_bytes(); - let position = memchr2(b'\n', b'\r', bytes)?; - - // SAFETY: memchr guarantees to return valid positions - #[allow(unsafe_code)] - let newline_character = unsafe { *bytes.get_unchecked(position) }; - - Some(match newline_character { - // Explicit branch for `\n` as this is the most likely path - b'\n' => (position, 1), - // '\r\n' - b'\r' if bytes.get(position.saturating_add(1)) == Some(&b'\n') => (position, 2), - // '\r' - _ => (position, 1), - }) -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use anyhow::Result; - use insta::assert_debug_snapshot; - - use crate::requirements::Requirements; - - #[test] - fn simple() -> Result<()> { - assert_debug_snapshot!(Requirements::from_str(r#"flask==2.0"#)?); - Ok(()) - } - - #[test] - fn pip_compile() -> Result<()> { - assert_debug_snapshot!(Requirements::from_str( - r" -# -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements.txt --resolver=backtracking pyproject.toml -# -attrs==23.1.0 \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 - # via - # cattrs - # lsprotocol -cattrs==23.1.2 \ - --hash=sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4 \ - --hash=sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657 - # via lsprotocol -exceptiongroup==1.1.3 \ - --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \ - --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3 - # via cattrs -importlib-metadata==6.7.0 \ - --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \ - --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5 - # via - # attrs - # typeguard -lsprotocol==2023.0.0b1 \ - --hash=sha256:ade2cd0fa0ede7965698cb59cd05d3adbd19178fd73e83f72ef57a032fbb9d62 \ - --hash=sha256:f7a2d4655cbd5639f373ddd1789807450c543341fa0a32b064ad30dbb9f510d4 - # via - # pygls - # ruff-lsp (pyproject.toml) -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 - # via ruff-lsp (pyproject.toml) -pygls==1.1.0 \ - --hash=sha256:70acb6fe0df1c8a17b7ce08daa0afdb4aedc6913a6a6696003e1434fda80a06e \ - --hash=sha256:eb19b818039d3d705ec8adbcdf5809a93af925f30cd7a3f3b7573479079ba00e - # via ruff-lsp (pyproject.toml) -ruff==0.0.292 \ - --hash=sha256:02f29db018c9d474270c704e6c6b13b18ed0ecac82761e4fcf0faa3728430c96 \ - --hash=sha256:1093449e37dd1e9b813798f6ad70932b57cf614e5c2b5c51005bf67d55db33ac \ - --hash=sha256:69654e564342f507edfa09ee6897883ca76e331d4bbc3676d8a8403838e9fade \ - --hash=sha256:6bdfabd4334684a4418b99b3118793f2c13bb67bf1540a769d7816410402a205 \ - --hash=sha256:6c3c91859a9b845c33778f11902e7b26440d64b9d5110edd4e4fa1726c41e0a4 \ - --hash=sha256:7f67a69c8f12fbc8daf6ae6d36705037bde315abf8b82b6e1f4c9e74eb750f68 \ - --hash=sha256:87616771e72820800b8faea82edd858324b29bb99a920d6aa3d3949dd3f88fb0 \ - --hash=sha256:8e087b24d0d849c5c81516ec740bf4fd48bf363cfb104545464e0fca749b6af9 \ - --hash=sha256:9889bac18a0c07018aac75ef6c1e6511d8411724d67cb879103b01758e110a81 \ - --hash=sha256:aa7c77c53bfcd75dbcd4d1f42d6cabf2485d2e1ee0678da850f08e1ab13081a8 \ - --hash=sha256:ac153eee6dd4444501c4bb92bff866491d4bfb01ce26dd2fff7ca472c8df9ad0 \ - --hash=sha256:b76deb3bdbea2ef97db286cf953488745dd6424c122d275f05836c53f62d4016 \ - --hash=sha256:be8eb50eaf8648070b8e58ece8e69c9322d34afe367eec4210fdee9a555e4ca7 \ - --hash=sha256:e854b05408f7a8033a027e4b1c7f9889563dd2aca545d13d06711e5c39c3d003 \ - --hash=sha256:f160b5ec26be32362d0774964e218f3fcf0a7da299f7e220ef45ae9e3e67101a \ - --hash=sha256:f27282bedfd04d4c3492e5c3398360c9d86a295be00eccc63914438b4ac8a83c \ - --hash=sha256:f4476f1243af2d8c29da5f235c13dca52177117935e1f9393f9d90f9833f69e4 - # via ruff-lsp (pyproject.toml) -typeguard==3.0.2 \ - --hash=sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e \ - --hash=sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a - # via pygls -typing-extensions==4.7.1 \ - --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ - --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 - # via - # cattrs - # importlib-metadata - # ruff-lsp (pyproject.toml) - # typeguard -zipp==3.15.0 \ - --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ - --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 - # via importlib-metadata -" - )?); - Ok(()) - } -} diff --git a/crates/puffin-package/src/requirements_txt.rs b/crates/puffin-package/src/requirements_txt.rs new file mode 100644 index 000000000..87c233b8a --- /dev/null +++ b/crates/puffin-package/src/requirements_txt.rs @@ -0,0 +1,582 @@ +//! Parses a subset of requirement.txt syntax +//! +//! +//! +//! Supported: +//! * [PEP 508 requirements](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) +//! * `-r` +//! * `-c` +//! * `--hash` (postfix) +//! * `-e` +//! +//! Unsupported: +//! * `-e `. TBD +//! * ``. TBD +//! * ``. TBD +//! * Options without a requirement, such as `--find-links` or `--index-url` +//! +//! Grammar as implemented: +//! +//! ```text +//! file = (statement | empty ('#' any*)? '\n')* +//! empty = whitespace* +//! statement = constraint_include | requirements_include | editable_requirement | requirement +//! constraint_include = '-c' ('=' | wrappable_whitespaces) filepath +//! requirements_include = '-r' ('=' | wrappable_whitespaces) filepath +//! editable_requirement = '-e' ('=' | wrappable_whitespaces) requirement +//! # We check whether the line starts with a letter or a number, in that case we assume it's a +//! # PEP 508 requirement +//! # https://packaging.python.org/en/latest/specifications/name-normalization/#valid-non-normalized-names +//! # This does not (yet?) support plain files or urls, we use a letter or a number as first +//! # character to assume a PEP 508 requirement +//! requirement = [a-zA-Z0-9] pep508_grammar_tail wrappable_whitespaces hashes +//! hashes = ('--hash' ('=' | wrappable_whitespaces) [a-zA-Z0-9-_]+ ':' [a-zA-Z0-9-_] wrappable_whitespaces+)* +//! # This should indicate a single backslash before a newline +//! wrappable_whitespaces = whitespace ('\\\n' | whitespace)* +//! ``` + +use std::fmt::{Display, Formatter}; +use std::io; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use fs_err as fs; +use serde::{Deserialize, Serialize}; +use tracing::warn; +use unscanny::{Pattern, Scanner}; + +use pep508_rs::{Pep508Error, Requirement}; + +/// We emit one of those for each requirements.txt entry +enum RequirementsTxtStatement { + /// `-r` inclusion filename + Requirements { + filename: String, + start: usize, + end: usize, + }, + /// `-c` inclusion filename + Constraint { + filename: String, + start: usize, + end: usize, + }, + /// PEP 508 requirement plus metadata + RequirementEntry(RequirementEntry), +} + +/// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in +/// the future also editable an similar information +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)] +pub struct RequirementEntry { + /// The actual PEP 508 requirement + pub requirement: Requirement, + /// Hashes of the downloadable packages + pub hashes: Vec, + /// Editable installation, see e.g. + pub editable: bool, +} + +impl Display for RequirementEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.editable { + write!(f, "-e ")?; + } + write!(f, "{}", self.requirement)?; + for hash in &self.hashes { + write!(f, " --hash {hash}")?; + } + Ok(()) + } +} + +/// Parsed and flattened requirements.txt with requirements and constraints +#[derive(Debug, Deserialize, Clone, Default, Eq, PartialEq, Serialize)] +pub struct RequirementsTxt { + /// The actual requirements with the hashes + pub requirements: Vec, + /// Constraints included with `-c` + pub constraints: Vec, +} + +impl RequirementsTxt { + /// See module level documentation + pub fn parse( + requirements_txt: impl AsRef, + working_dir: impl AsRef, + ) -> Result { + let content = + fs::read_to_string(&requirements_txt).map_err(|err| RequirementsTxtFileError { + file: requirements_txt.as_ref().to_path_buf(), + error: RequirementsTxtParserError::IO(err), + })?; + let data = + Self::parse_inner(&content, working_dir).map_err(|err| RequirementsTxtFileError { + file: requirements_txt.as_ref().to_path_buf(), + error: err, + })?; + if data == Self::default() { + warn!( + "Requirements file {} does not contain any dependencies", + requirements_txt.as_ref().display() + ); + } + Ok(data) + } + + /// See module level documentation + /// + /// Note that all relative paths are dependent on the current working dir, not on the location + /// of the file + pub fn parse_inner( + content: &str, + working_dir: impl AsRef, + ) -> Result { + let mut s = Scanner::new(content); + + let mut data = Self::default(); + while let Some(statement) = parse_entry(&mut s, content)? { + match statement { + RequirementsTxtStatement::Requirements { + filename, + start, + end, + } => { + let sub_file = working_dir.as_ref().join(filename); + let sub_requirements = + Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { + RequirementsTxtParserError::Subfile { + source: Box::new(err), + start, + end, + } + })?; + // Add each to the correct category + data.update_from(sub_requirements); + } + RequirementsTxtStatement::Constraint { + filename, + start, + end, + } => { + let sub_file = working_dir.as_ref().join(filename); + let sub_constraints = + Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { + RequirementsTxtParserError::Subfile { + source: Box::new(err), + start, + end, + } + })?; + // Here we add both to constraints + data.constraints.extend( + sub_constraints + .requirements + .into_iter() + .map(|requirement_entry| requirement_entry.requirement), + ); + data.constraints.extend(sub_constraints.constraints); + } + RequirementsTxtStatement::RequirementEntry(requirement_entry) => { + data.requirements.push(requirement_entry); + } + } + } + Ok(data) + } + + /// Merges other into self + pub fn update_from(&mut self, other: RequirementsTxt) { + self.requirements.extend(other.requirements); + self.constraints.extend(other.constraints); + } +} + +/// Parse a single entry, that is a requirement, an inclusion or a comment line +/// +/// Consumes all preceding trivia (whitespace and comments). If it returns None, we've reached +/// the end of file +fn parse_entry( + s: &mut Scanner, + content: &str, +) -> Result, RequirementsTxtParserError> { + // Eat all preceding whitespace, this may run us to the end of file + eat_wrappable_whitespace(s); + while s.at(['\n', '\r', '#']) { + // skip comments + eat_trailing_line(s)?; + eat_wrappable_whitespace(s); + } + + let start = s.cursor(); + Ok(Some(if s.eat_if("-r") { + let requirements_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; + let end = s.cursor(); + eat_trailing_line(s)?; + RequirementsTxtStatement::Requirements { + filename: requirements_file.to_string(), + start, + end, + } + } else if s.eat_if("-c") { + let constraints_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; + let end = s.cursor(); + eat_trailing_line(s)?; + RequirementsTxtStatement::Constraint { + filename: constraints_file.to_string(), + start, + end, + } + } else if s.eat_if("-e") { + let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; + RequirementsTxtStatement::RequirementEntry(RequirementEntry { + requirement, + hashes, + editable: true, + }) + } else if s.at(char::is_ascii_alphanumeric) { + let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; + RequirementsTxtStatement::RequirementEntry(RequirementEntry { + requirement, + hashes, + editable: false, + }) + } else if let Some(char) = s.peek() { + return Err(RequirementsTxtParserError::Parser { + message: format!( + "Unexpected '{char}', expected '-c', '-e', '-r' or the start of a requirement" + ), + location: s.cursor(), + }); + } else { + // EOF + return Ok(None); + })) +} + +/// Eat whitespace and ignore newlines escaped with a backslash +fn eat_wrappable_whitespace<'a>(s: &mut Scanner<'a>) -> &'a str { + let start = s.cursor(); + s.eat_while([' ', '\t']); + // Allow multiple escaped line breaks + // With the order we support `\n`, `\r`, `\r\n` without accidentally eating a `\n\r` + while s.eat_if("\\\n") || s.eat_if("\\\r\n") || s.eat_if("\\\r") { + s.eat_while([' ', '\t']); + } + s.from(start) +} + +/// Eats the end of line or a potential trailing comma +fn eat_trailing_line(s: &mut Scanner) -> Result<(), RequirementsTxtParserError> { + s.eat_while([' ', '\t']); + match s.eat() { + None | Some('\n') => {} // End of file or end of line, nothing to do + Some('\r') => { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + Some('#') => { + s.eat_until(['\r', '\n']); + if s.at('\r') { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + } + Some(other) => { + return Err(RequirementsTxtParserError::Parser { + message: format!("Expected comment or end-of-line, found '{other}'"), + location: s.cursor(), + }); + } + } + Ok(()) +} + +/// Parse a PEP 508 requirement with optional trailing hashes +fn parse_requirement_and_hashes( + s: &mut Scanner, + content: &str, +) -> Result<(Requirement, Vec), RequirementsTxtParserError> { + // PEP 508 requirement + let start = s.cursor(); + // Termination: s.eat() eventually becomes None + let (end, has_hashes) = loop { + let end = s.cursor(); + + // We look for the end of the line ... + if s.eat_if('\n') { + break (end, false); + } + if s.eat_if('\r') { + s.eat_if('\n'); // Support `\r\n` but also accept stray `\r` + break (end, false); + } + // ... or `--hash`, an escaped newline or a comment separated by whitespace ... + if !eat_wrappable_whitespace(s).is_empty() { + if s.after().starts_with("--") { + break (end, true); + } else if s.eat_if('#') { + s.eat_until(['\r', '\n']); + if s.at('\r') { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + break (end, false); + } + continue; + } + // ... or the end of the file, which works like the end of line + if s.eat().is_none() { + break (end, false); + } + }; + let requirement = Requirement::from_str(&content[start..end]).map_err(|err| { + RequirementsTxtParserError::Pep508 { + source: err, + start, + end, + } + })?; + let hashes = if has_hashes { + let hashes = parse_hashes(s)?; + eat_trailing_line(s)?; + hashes + } else { + Vec::new() + }; + Ok((requirement, hashes)) +} + +/// Parse `--hash=... --hash ...` after a requirement +fn parse_hashes(s: &mut Scanner) -> Result, RequirementsTxtParserError> { + let mut hashes = Vec::new(); + if s.eat_while("--hash").is_empty() { + return Err(RequirementsTxtParserError::Parser { + message: format!( + "Expected '--hash', found '{:?}'", + s.eat_while(|c: char| !c.is_whitespace()) + ), + location: s.cursor(), + }); + } + let hash = parse_value(s, |c: char| !c.is_whitespace())?; + hashes.push(hash.to_string()); + loop { + eat_wrappable_whitespace(s); + if !s.eat_if("--hash") { + break; + } + let hash = parse_value(s, |c: char| !c.is_whitespace())?; + hashes.push(hash.to_string()); + } + Ok(hashes) +} + +/// In `-=` or `- value`, this parses the part after the key +fn parse_value<'a, T>( + s: &mut Scanner<'a>, + while_pattern: impl Pattern, +) -> Result<&'a str, RequirementsTxtParserError> { + if s.eat_if('=') { + // Explicit equals sign + Ok(s.eat_while(while_pattern).trim_end()) + } else if s.eat_if(char::is_whitespace) { + // Key and value are separated by whitespace instead + s.eat_whitespace(); + Ok(s.eat_while(while_pattern).trim_end()) + } else { + Err(RequirementsTxtParserError::Parser { + message: format!("Expected '=' or whitespace, found {:?}", s.peek()), + location: s.cursor(), + }) + } +} + +/// Error parsing requirements.txt, wrapper with filename +#[derive(Debug)] +pub struct RequirementsTxtFileError { + file: PathBuf, + error: RequirementsTxtParserError, +} + +/// Error parsing requirements.txt, error disambiguation +#[derive(Debug)] +pub enum RequirementsTxtParserError { + IO(io::Error), + Parser { + message: String, + location: usize, + }, + Pep508 { + source: Pep508Error, + start: usize, + end: usize, + }, + Subfile { + source: Box, + start: usize, + end: usize, + }, +} + +impl Display for RequirementsTxtFileError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self.error { + RequirementsTxtParserError::IO(err) => err.fmt(f), + RequirementsTxtParserError::Parser { message, location } => { + write!( + f, + "{} in {} position {}", + message, + self.file.display(), + location + ) + } + RequirementsTxtParserError::Pep508 { start, end, .. } => { + write!( + f, + "Couldn't parse requirement in {} position {} to {}", + self.file.display(), + start, + end, + ) + } + RequirementsTxtParserError::Subfile { start, end, .. } => { + write!( + f, + "Error parsing file included into {} at position {} to {}", + self.file.display(), + start, + end + ) + } + } + } +} + +impl std::error::Error for RequirementsTxtFileError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match &self.error { + RequirementsTxtParserError::IO(err) => err.source(), + RequirementsTxtParserError::Pep508 { source, .. } => Some(source), + RequirementsTxtParserError::Subfile { source, .. } => Some(source.as_ref()), + RequirementsTxtParserError::Parser { .. } => None, + } + } +} + +#[cfg(test)] +mod test { + use std::path::{Path, PathBuf}; + + use fs_err as fs; + use indoc::indoc; + use tempfile::tempdir; + use test_case::test_case; + + use crate::requirements_txt::RequirementsTxt; + + #[test_case(Path::new("basic.txt"))] + #[test_case(Path::new("constraints-a.txt"))] + #[test_case(Path::new("constraints-b.txt"))] + #[test_case(Path::new("empty.txt"))] + #[test_case(Path::new("for-poetry.txt"))] + #[test_case(Path::new("include-a.txt"))] + #[test_case(Path::new("include-b.txt"))] + #[test_case(Path::new("poetry-with-hashes.txt"))] + #[test_case(Path::new("small.txt"))] + #[test_case(Path::new("whitespace.txt"))] + fn parse(path: &Path) { + let working_dir = workspace_test_data_dir().join("requirements-txt"); + let requirements_txt = working_dir.join(path); + + let actual = RequirementsTxt::parse(&requirements_txt, &working_dir).unwrap(); + + let snapshot = format!("parse-{}", path.to_string_lossy()); + insta::assert_debug_snapshot!(snapshot, actual); + } + + #[test_case(Path::new("basic.txt"))] + #[test_case(Path::new("constraints-a.txt"))] + #[test_case(Path::new("constraints-b.txt"))] + #[test_case(Path::new("empty.txt"))] + #[test_case(Path::new("for-poetry.txt"))] + #[test_case(Path::new("include-a.txt"))] + #[test_case(Path::new("include-b.txt"))] + #[test_case(Path::new("poetry-with-hashes.txt"))] + #[test_case(Path::new("small.txt"))] + #[test_case(Path::new("whitespace.txt"))] + fn line_endings(path: &Path) { + let working_dir = workspace_test_data_dir().join("requirements-txt"); + let requirements_txt = working_dir.join(path); + + // Replace line endings with the other choice. This works even if you use git with LF + // only on windows. + let contents = fs::read_to_string(requirements_txt).unwrap(); + let contents = if contents.contains("\r\n") { + contents.replace("\r\n", "\n") + } else { + contents.replace('\n', "\r\n") + }; + + // Write to a new file. + let temp_dir = tempdir().unwrap(); + let requirements_txt = temp_dir.path().join(path); + fs::write(&requirements_txt, &contents).unwrap(); + + let actual = RequirementsTxt::parse(&requirements_txt, &working_dir).unwrap(); + + let snapshot = format!("line-endings-{}", path.to_string_lossy()); + insta::assert_debug_snapshot!(snapshot, actual); + } + + #[test] + fn invalid_include_missing_file() { + let working_dir = workspace_test_data_dir().join("requirements-txt"); + let basic = working_dir.join("invalid-include"); + let missing = working_dir.join("missing.txt"); + let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); + let errors = anyhow::Error::new(err) + .chain() + .map(ToString::to_string) + .collect::>(); + assert_eq!(errors.len(), 3); + assert_eq!( + errors[0], + format!( + "Error parsing file included into {} at position 0 to 14", + basic.display() + ) + ); + assert_eq!( + errors[1], + format!("failed to open file `{}`", missing.display()), + ); + // The last error message is os specific + } + + #[test] + fn invalid_requirement() { + let working_dir = workspace_test_data_dir().join("requirements-txt"); + let basic = working_dir.join("invalid-requirement"); + let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); + let errors = anyhow::Error::new(err) + .chain() + .map(ToString::to_string) + .collect::>(); + let expected = &[ + format!( + "Couldn't parse requirement in {} position 0 to 15", + basic.display() + ), + indoc! {" + Expected an alphanumeric character starting the extra name, found 'ö' + numpy[ö]==1.29 + ^" + } + .to_string(), + ]; + assert_eq!(errors, expected); + } + + fn workspace_test_data_dir() -> PathBuf { + PathBuf::from("./test-data") + } +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__pip_compile.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__pip_compile.snap deleted file mode 100644 index d532659be..000000000 --- a/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__pip_compile.snap +++ /dev/null @@ -1,320 +0,0 @@ ---- -source: crates/puffin-package/src/requirements.rs -expression: "Requirements::from_str(r\"\n#\n# This file is autogenerated by pip-compile with Python 3.7\n# by the following command:\n#\n# pip-compile --generate-hashes --output-file=requirements.txt --resolver=backtracking pyproject.toml\n#\nattrs==23.1.0 \\\n --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \\\n --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015\n # via\n # cattrs\n # lsprotocol\ncattrs==23.1.2 \\\n --hash=sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4 \\\n --hash=sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657\n # via lsprotocol\nexceptiongroup==1.1.3 \\\n --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \\\n --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3\n # via cattrs\nimportlib-metadata==6.7.0 \\\n --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \\\n --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5\n # via\n # attrs\n # typeguard\nlsprotocol==2023.0.0b1 \\\n --hash=sha256:ade2cd0fa0ede7965698cb59cd05d3adbd19178fd73e83f72ef57a032fbb9d62 \\\n --hash=sha256:f7a2d4655cbd5639f373ddd1789807450c543341fa0a32b064ad30dbb9f510d4\n # via\n # pygls\n # ruff-lsp (pyproject.toml)\npackaging==23.2 \\\n --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \\\n --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7\n # via ruff-lsp (pyproject.toml)\npygls==1.1.0 \\\n --hash=sha256:70acb6fe0df1c8a17b7ce08daa0afdb4aedc6913a6a6696003e1434fda80a06e \\\n --hash=sha256:eb19b818039d3d705ec8adbcdf5809a93af925f30cd7a3f3b7573479079ba00e\n # via ruff-lsp (pyproject.toml)\nruff==0.0.292 \\\n --hash=sha256:02f29db018c9d474270c704e6c6b13b18ed0ecac82761e4fcf0faa3728430c96 \\\n --hash=sha256:1093449e37dd1e9b813798f6ad70932b57cf614e5c2b5c51005bf67d55db33ac \\\n --hash=sha256:69654e564342f507edfa09ee6897883ca76e331d4bbc3676d8a8403838e9fade \\\n --hash=sha256:6bdfabd4334684a4418b99b3118793f2c13bb67bf1540a769d7816410402a205 \\\n --hash=sha256:6c3c91859a9b845c33778f11902e7b26440d64b9d5110edd4e4fa1726c41e0a4 \\\n --hash=sha256:7f67a69c8f12fbc8daf6ae6d36705037bde315abf8b82b6e1f4c9e74eb750f68 \\\n --hash=sha256:87616771e72820800b8faea82edd858324b29bb99a920d6aa3d3949dd3f88fb0 \\\n --hash=sha256:8e087b24d0d849c5c81516ec740bf4fd48bf363cfb104545464e0fca749b6af9 \\\n --hash=sha256:9889bac18a0c07018aac75ef6c1e6511d8411724d67cb879103b01758e110a81 \\\n --hash=sha256:aa7c77c53bfcd75dbcd4d1f42d6cabf2485d2e1ee0678da850f08e1ab13081a8 \\\n --hash=sha256:ac153eee6dd4444501c4bb92bff866491d4bfb01ce26dd2fff7ca472c8df9ad0 \\\n --hash=sha256:b76deb3bdbea2ef97db286cf953488745dd6424c122d275f05836c53f62d4016 \\\n --hash=sha256:be8eb50eaf8648070b8e58ece8e69c9322d34afe367eec4210fdee9a555e4ca7 \\\n --hash=sha256:e854b05408f7a8033a027e4b1c7f9889563dd2aca545d13d06711e5c39c3d003 \\\n --hash=sha256:f160b5ec26be32362d0774964e218f3fcf0a7da299f7e220ef45ae9e3e67101a \\\n --hash=sha256:f27282bedfd04d4c3492e5c3398360c9d86a295be00eccc63914438b4ac8a83c \\\n --hash=sha256:f4476f1243af2d8c29da5f235c13dca52177117935e1f9393f9d90f9833f69e4\n # via ruff-lsp (pyproject.toml)\ntypeguard==3.0.2 \\\n --hash=sha256:bbe993854385284ab42fd5bd3bee6f6556577ce8b50696d6cb956d704f286c8e \\\n --hash=sha256:fee5297fdb28f8e9efcb8142b5ee219e02375509cd77ea9d270b5af826358d5a\n # via pygls\ntyping-extensions==4.7.1 \\\n --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \\\n --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2\n # via\n # cattrs\n # importlib-metadata\n # ruff-lsp (pyproject.toml)\n # typeguard\nzipp==3.15.0 \\\n --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \\\n --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556\n # via importlib-metadata\n\")?" ---- -Requirements( - [ - Requirement { - name: "attrs", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 23, - 1, - 0, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "cattrs", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 23, - 1, - 2, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "exceptiongroup", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 1, - 1, - 3, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "importlib-metadata", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 6, - 7, - 0, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "lsprotocol", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 2023, - 0, - 0, - ], - pre: Some( - ( - Beta, - 1, - ), - ), - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "packaging", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 23, - 2, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "pygls", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 1, - 1, - 0, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "ruff", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 0, - 0, - 292, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "typeguard", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 3, - 0, - 2, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "typing-extensions", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 4, - 7, - 1, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - Requirement { - name: "zipp", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 3, - 15, - 0, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - ], -) diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__simple.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__simple.snap deleted file mode 100644 index d6ce2d9de..000000000 --- a/crates/puffin-package/src/snapshots/puffin_package__requirements__tests__simple.snap +++ /dev/null @@ -1,35 +0,0 @@ ---- -source: crates/puffin-package/src/requirements.rs -expression: "Requirements::from_str(r#\"flask==2.0\"#)?" ---- -Requirements( - [ - Requirement { - name: "flask", - extras: None, - version_or_url: Some( - VersionSpecifier( - VersionSpecifiers( - [ - VersionSpecifier { - operator: Equal, - version: Version { - epoch: 0, - release: [ - 2, - 0, - ], - pre: None, - post: None, - dev: None, - local: None, - }, - }, - ], - ), - ), - ), - marker: None, - }, - ], -) diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-basic.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-basic.txt.snap new file mode 100644 index 000000000..034089301 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-basic.txt.snap @@ -0,0 +1,199 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 24, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 0, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "python-dateutil", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 8, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "six", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 16, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "tzdata", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-a.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-a.txt.snap new file mode 100644 index 000000000..1b37888d0 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-a.txt.snap @@ -0,0 +1,96 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "django-debug-toolbar", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: LessThan, + version: Version { + epoch: 0, + release: [ + 2, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [ + Requirement { + name: "django", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 1, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + ], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-b.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-b.txt.snap new file mode 100644 index 000000000..d83fe87ac --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-constraints-b.txt.snap @@ -0,0 +1,72 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "django", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 1, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-empty.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-empty.txt.snap new file mode 100644 index 000000000..7ed4f9cba --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-empty.txt.snap @@ -0,0 +1,8 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-for-poetry.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-for-poetry.txt.snap new file mode 100644 index 000000000..c60fd9e4e --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-for-poetry.txt.snap @@ -0,0 +1,129 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "inflection", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 0, + 5, + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "upsidedown", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 0, + 4, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: Some( + [ + "tabulate", + ], + ), + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: Version { + epoch: 0, + release: [ + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + VersionSpecifier { + operator: LessThan, + version: Version { + epoch: 0, + release: [ + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-a.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-a.txt.snap new file mode 100644 index 000000000..c590fb118 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-a.txt.snap @@ -0,0 +1,51 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tomli", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 24, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-b.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-b.txt.snap new file mode 100644 index 000000000..0c97db984 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-include-b.txt.snap @@ -0,0 +1,19 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tomli", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-poetry-with-hashes.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-poetry-with-hashes.txt.snap new file mode 100644 index 000000000..b7c48b752 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-poetry-with-hashes.txt.snap @@ -0,0 +1,329 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "werkzeug", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 2, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "urllib3", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 26, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "ansicon", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 89, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvString( + PlatformSystem, + ), + operator: Equal, + r_value: QuotedString( + "Windows", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "requests-oauthlib", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 3, + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", + "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "psycopg2", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 9, + 5, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955", + "sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa", + "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e", + "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a", + ], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-small.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-small.txt.snap new file mode 100644 index 000000000..636e5084d --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-small.txt.snap @@ -0,0 +1,73 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tqdm", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 4, + 65, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "tomli-w", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 0, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-whitespace.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-whitespace.txt.snap new file mode 100644 index 000000000..9d81dfa55 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__line-endings-whitespace.txt.snap @@ -0,0 +1,51 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: Some( + [ + "tabulate", + ], + ), + version_or_url: Some( + Url( + Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "github.com", + ), + ), + port: None, + path: "/pandas-dev/pandas", + query: None, + fragment: None, + }, + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-basic.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-basic.txt.snap new file mode 100644 index 000000000..034089301 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-basic.txt.snap @@ -0,0 +1,199 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 24, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 0, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "python-dateutil", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 8, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "six", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 16, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "tzdata", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-a.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-a.txt.snap new file mode 100644 index 000000000..1b37888d0 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-a.txt.snap @@ -0,0 +1,96 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "django-debug-toolbar", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: LessThan, + version: Version { + epoch: 0, + release: [ + 2, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [ + Requirement { + name: "django", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 1, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + ], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-b.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-b.txt.snap new file mode 100644 index 000000000..d83fe87ac --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-constraints-b.txt.snap @@ -0,0 +1,72 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "django", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 1, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pytz", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2023, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-empty.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-empty.txt.snap new file mode 100644 index 000000000..7ed4f9cba --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-empty.txt.snap @@ -0,0 +1,8 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-for-poetry.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-for-poetry.txt.snap new file mode 100644 index 000000000..c60fd9e4e --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-for-poetry.txt.snap @@ -0,0 +1,129 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "inflection", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 0, + 5, + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "upsidedown", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 0, + 4, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: Some( + [ + "tabulate", + ], + ), + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: Version { + epoch: 0, + release: [ + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + VersionSpecifier { + operator: LessThan, + version: Version { + epoch: 0, + release: [ + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-a.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-a.txt.snap new file mode 100644 index 000000000..c590fb118 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-a.txt.snap @@ -0,0 +1,51 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tomli", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 24, + 2, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-b.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-b.txt.snap new file mode 100644 index 000000000..0c97db984 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-include-b.txt.snap @@ -0,0 +1,19 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tomli", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-poetry-with-hashes.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-poetry-with-hashes.txt.snap new file mode 100644 index 000000000..b7c48b752 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-poetry-with-hashes.txt.snap @@ -0,0 +1,329 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "werkzeug", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 2, + 3, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "urllib3", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 26, + 15, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "ansicon", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 89, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvString( + PlatformSystem, + ), + operator: Equal, + r_value: QuotedString( + "Windows", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "requests-oauthlib", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 3, + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", + "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a", + ], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "psycopg2", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 2, + 9, + 5, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: Some( + And( + [ + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: GreaterEqual, + r_value: QuotedString( + "3.8", + ), + }, + ), + Expression( + MarkerExpression { + l_value: MarkerEnvVersion( + PythonVersion, + ), + operator: LessThan, + r_value: QuotedString( + "4.0", + ), + }, + ), + ], + ), + ), + }, + hashes: [ + "sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955", + "sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa", + "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e", + "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a", + ], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-small.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-small.txt.snap new file mode 100644 index 000000000..636e5084d --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-small.txt.snap @@ -0,0 +1,73 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "tqdm", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 4, + 65, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "tomli-w", + extras: None, + version_or_url: Some( + VersionSpecifier( + VersionSpecifiers( + [ + VersionSpecifier { + operator: Equal, + version: Version { + epoch: 0, + release: [ + 1, + 0, + 0, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-whitespace.txt.snap b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-whitespace.txt.snap new file mode 100644 index 000000000..9d81dfa55 --- /dev/null +++ b/crates/puffin-package/src/snapshots/puffin_package__requirements_txt__test__parse-whitespace.txt.snap @@ -0,0 +1,51 @@ +--- +source: crates/puffin-package/src/requirements_txt.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: "numpy", + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: "pandas", + extras: Some( + [ + "tabulate", + ], + ), + version_or_url: Some( + Url( + Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "github.com", + ), + ), + port: None, + path: "/pandas-dev/pandas", + query: None, + fragment: None, + }, + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], +} diff --git a/crates/puffin-package/test-data/requirements-txt/basic.txt b/crates/puffin-package/test-data/requirements-txt/basic.txt new file mode 100644 index 000000000..74d0230f4 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/basic.txt @@ -0,0 +1,6 @@ +numpy==1.24.2 +pandas==2.0.0 +python-dateutil==2.8.2 +pytz==2023.3 +six==1.16.0 +tzdata==2023.3 \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/constraints-a.txt b/crates/puffin-package/test-data/requirements-txt/constraints-a.txt new file mode 100644 index 000000000..efe55d468 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/constraints-a.txt @@ -0,0 +1,2 @@ +-c constraints-b.txt # We can't actually semantically deal with those yet +django-debug-toolbar<2.2 \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/constraints-b.txt b/crates/puffin-package/test-data/requirements-txt/constraints-b.txt new file mode 100644 index 000000000..61214b358 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/constraints-b.txt @@ -0,0 +1,2 @@ +django==2.1.15 +pytz==2023.3 \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/empty.txt b/crates/puffin-package/test-data/requirements-txt/empty.txt new file mode 100644 index 000000000..e69de29bb diff --git a/crates/puffin-package/test-data/requirements-txt/for-poetry.txt b/crates/puffin-package/test-data/requirements-txt/for-poetry.txt new file mode 100644 index 000000000..c0b7401b5 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/for-poetry.txt @@ -0,0 +1,6 @@ + +# Used in requirements_txt_to_poetry +inflection==0.5.1 +upsidedown==0.4 +numpy +pandas[tabulate]>=1,<2 diff --git a/crates/puffin-package/test-data/requirements-txt/include-a.txt b/crates/puffin-package/test-data/requirements-txt/include-a.txt new file mode 100644 index 000000000..107116d78 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/include-a.txt @@ -0,0 +1,3 @@ +-r include-b.txt + +numpy==1.24.2 diff --git a/crates/puffin-package/test-data/requirements-txt/include-b.txt b/crates/puffin-package/test-data/requirements-txt/include-b.txt new file mode 100644 index 000000000..b6856cb9a --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/include-b.txt @@ -0,0 +1 @@ +tomli \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/invalid-include b/crates/puffin-package/test-data/requirements-txt/invalid-include new file mode 100644 index 000000000..309cbd45e --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/invalid-include @@ -0,0 +1 @@ +-r missing.txt \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/invalid-requirement b/crates/puffin-package/test-data/requirements-txt/invalid-requirement new file mode 100644 index 000000000..69dc80962 --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/invalid-requirement @@ -0,0 +1 @@ +numpy[ö]==1.29 \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/poetry-with-hashes.txt b/crates/puffin-package/test-data/requirements-txt/poetry-with-hashes.txt new file mode 100644 index 000000000..a278f8d7d --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/poetry-with-hashes.txt @@ -0,0 +1,14 @@ +# Includes more styles than poetry uses +werkzeug==2.2.3 ; python_version >= "3.8" and python_version < "4.0" --hash=sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe +urllib3==1.26.15 ; python_version >= "3.8" and python_version < "4" \ + --hash sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 +ansicon==1.89.0 ; python_version >= "3.8" and python_version < "4" and platform_system == "Windows" \ + --hash=sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1 +requests-oauthlib==1.3.1 ; python_version >= "3.8" and python_version < "4.0" \ + --hash=sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5 \ + --hash=sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a +psycopg2==2.9.5 ; python_version >= "3.8" and python_version < "4.0" \ + --hash=sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955 \ + --hash=sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa \ + --hash=sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e \ + --hash=sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a diff --git a/crates/puffin-package/test-data/requirements-txt/small.txt b/crates/puffin-package/test-data/requirements-txt/small.txt new file mode 100644 index 000000000..7b973e90a --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/small.txt @@ -0,0 +1,4 @@ +# These are small and fast to install + +tqdm==4.65.0 +tomli-w==1.0.0 \ No newline at end of file diff --git a/crates/puffin-package/test-data/requirements-txt/whitespace.txt b/crates/puffin-package/test-data/requirements-txt/whitespace.txt new file mode 100644 index 000000000..c1e99017e --- /dev/null +++ b/crates/puffin-package/test-data/requirements-txt/whitespace.txt @@ -0,0 +1,25 @@ + + + # # + # # + # # + # # + # # + # # + # # + + ## + +# + + numpy # # + +\ + +pandas [tabulate] @ https://github.com/pandas-dev/pandas \ + # üh + + + # + # 안녕 + #