Add Poetry support to bench.py (#803)

## Summary

Enables benchmarking against Poetry for resolution and installation:

```
Benchmark 1: pip-tools (resolve-cold)
  Time (mean ± σ):     962.7 ms ± 241.9 ms    [User: 322.8 ms, System: 80.5 ms]
  Range (min … max):   714.9 ms … 1459.4 ms    10 runs

Benchmark 1: puffin (resolve-cold)
  Time (mean ± σ):     193.2 ms ±   8.2 ms    [User: 31.3 ms, System: 22.8 ms]
  Range (min … max):   179.8 ms … 206.4 ms    14 runs

Benchmark 1: poetry (resolve-cold)
  Time (mean ± σ):     900.7 ms ±  21.2 ms    [User: 371.6 ms, System: 92.1 ms]
  Range (min … max):   855.7 ms … 933.4 ms    10 runs

Benchmark 1: pip-tools (resolve-warm)
  Time (mean ± σ):     386.0 ms ±  19.1 ms    [User: 255.8 ms, System: 46.2 ms]
  Range (min … max):   368.7 ms … 434.5 ms    10 runs

Benchmark 1: puffin (resolve-warm)
  Time (mean ± σ):       8.1 ms ±   0.4 ms    [User: 4.4 ms, System: 5.1 ms]
  Range (min … max):     7.5 ms …  11.1 ms    183 runs

Benchmark 1: poetry (resolve-warm)
  Time (mean ± σ):     336.3 ms ±   0.6 ms    [User: 283.6 ms, System: 44.7 ms]
  Range (min … max):   335.0 ms … 337.3 ms    10 runs
```

scripts/bench/__main__.py

@@ -1,10 +1,20 @@
 """Benchmark Puffin against other packaging tools.
 
-This script assumes that `pip`, `pip-tools`, `virtualenv`, and `hyperfine` are
+This script assumes that `pip`, `pip-tools`, `virtualenv`, `poetry` and `hyperfine` are
 installed, and that a Puffin release builds exists at `./target/release/puffin`
 (relative to the repository root).
 
-Example usage: python bench.py -t puffin -t pip-tools requirements.in
+This script assumes that Python 3.10 is installed.
+
+To set up the required environment, run:
+
+    cargo build --release
+    ./target/release/puffin venv
+    ./target/release/puffin pip-sync ./scripts/requirements.txt
+
+Example usage:
+
+    python -m scripts.bench -t puffin -t pip-tools requirements.in
 """
 import abc
 import argparse
@@ -15,6 +25,10 @@ import shlex
 import subprocess
 import tempfile
 
+import tomli
+import tomli_w
+from packaging.requirements import Requirement
+
 WARMUP = 3
 MIN_RUNS = 10
 
@@ -24,6 +38,7 @@ class Tool(enum.Enum):
 
     PIP_TOOLS = "pip-tools"
     PUFFIN = "puffin"
+    POETRY = "poetry"
 
 
 class Benchmark(enum.Enum):
@@ -364,6 +379,231 @@ class Puffin(Suite):
             )
 
 
+class Poetry(Suite):
+    def init(self, requirements_file: str, *, working_dir: str) -> None:
+        """Initialize a Poetry project from a requirements file."""
+        # Parse all dependencies from the requirements file.
+        with open(requirements_file) as fp:
+            requirements = [
+                Requirement(line) for line in fp if not line.startswith("#")
+            ]
+
+        # Create a Poetry project.
+        subprocess.check_call(
+            [
+                "poetry",
+                "init",
+                "--name",
+                "bench",
+                "--no-interaction",
+                "--python",
+                ">=3.10",
+            ],
+            cwd=working_dir,
+        )
+
+        # Parse the pyproject.toml.
+        with open(os.path.join(working_dir, "pyproject.toml"), "rb") as fp:
+            pyproject = tomli.load(fp)
+
+        # Add the dependencies to the pyproject.toml.
+        pyproject["tool"]["poetry"]["dependencies"].update(
+            {
+                str(requirement.name): str(requirement.specifier)
+                if requirement.specifier
+                else "*"
+                for requirement in requirements
+            }
+        )
+
+        with open(os.path.join(working_dir, "pyproject.toml"), "wb") as fp:
+            tomli_w.dump(pyproject, fp)
+
+    def resolve_cold(self, requirements_file: str, *, verbose: bool) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.init(requirements_file, working_dir=temp_dir)
+
+            poetry_lock = os.path.join(temp_dir, "poetry.lock")
+            config_dir = os.path.join(temp_dir, "config", "pypoetry")
+            cache_dir = os.path.join(temp_dir, "cache", "pypoetry")
+            data_dir = os.path.join(temp_dir, "data", "pypoetry")
+
+            subprocess.check_call(
+                [
+                    "hyperfine",
+                    *(["--show-output"] if verbose else []),
+                    "--command-name",
+                    f"{Tool.POETRY.value} ({Benchmark.RESOLVE_COLD.value})",
+                    "--warmup",
+                    str(WARMUP),
+                    "--min-runs",
+                    str(MIN_RUNS),
+                    "--prepare",
+                    (
+                        f"rm -rf {config_dir} && "
+                        f"rm -rf {cache_dir} && "
+                        f"rm -rf {data_dir} &&"
+                        f"rm -rf {poetry_lock}"
+                    ),
+                    shlex.join(
+                        [
+                            f"POETRY_CONFIG_DIR={config_dir}",
+                            f"POETRY_CACHE_DIR={cache_dir}",
+                            f"POETRY_DATA_DIR={data_dir}",
+                            "poetry",
+                            "lock",
+                        ]
+                    ),
+                ],
+                cwd=temp_dir,
+            )
+
+    def resolve_warm(self, requirements_file: str, *, verbose: bool) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.init(requirements_file, working_dir=temp_dir)
+
+            poetry_lock = os.path.join(temp_dir, "poetry.lock")
+            config_dir = os.path.join(temp_dir, "config", "pypoetry")
+            cache_dir = os.path.join(temp_dir, "cache", "pypoetry")
+            data_dir = os.path.join(temp_dir, "data", "pypoetry")
+
+            subprocess.check_call(
+                [
+                    "hyperfine",
+                    *(["--show-output"] if verbose else []),
+                    "--command-name",
+                    f"{Tool.POETRY.value} ({Benchmark.RESOLVE_WARM.value})",
+                    "--warmup",
+                    str(WARMUP),
+                    "--min-runs",
+                    str(MIN_RUNS),
+                    "--prepare",
+                    f"rm -rf {poetry_lock}",
+                    shlex.join(
+                        [
+                            f"POETRY_CONFIG_DIR={config_dir}",
+                            f"POETRY_CACHE_DIR={cache_dir}",
+                            f"POETRY_DATA_DIR={data_dir}",
+                            "poetry",
+                            "lock",
+                        ]
+                    ),
+                ],
+                cwd=temp_dir,
+            )
+
+    def install_cold(self, requirements_file: str, *, verbose: bool) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.init(requirements_file, working_dir=temp_dir)
+
+            poetry_lock = os.path.join(temp_dir, "poetry.lock")
+            assert not os.path.exists(
+                poetry_lock
+            ), f"Lock file already exists at: {poetry_lock}"
+
+            # Run a resolution, to ensure that the lock file exists.
+            subprocess.check_call(
+                ["poetry", "lock"],
+                cwd=temp_dir,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            assert os.path.exists(
+                poetry_lock
+            ), f"Lock file doesn't exist at: {poetry_lock}"
+
+            config_dir = os.path.join(temp_dir, "config", "pypoetry")
+            cache_dir = os.path.join(temp_dir, "cache", "pypoetry")
+            data_dir = os.path.join(temp_dir, "data", "pypoetry")
+            venv_dir = os.path.join(temp_dir, ".venv")
+
+            subprocess.check_call(
+                [
+                    "hyperfine",
+                    *(["--show-output"] if verbose else []),
+                    "--command-name",
+                    f"{Tool.POETRY.value} ({Benchmark.INSTALL_COLD.value})",
+                    "--warmup",
+                    str(WARMUP),
+                    "--min-runs",
+                    str(MIN_RUNS),
+                    "--prepare",
+                    (
+                        f"rm -rf {config_dir} && "
+                        f"rm -rf {cache_dir} && "
+                        f"rm -rf {data_dir} &&"
+                        f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
+                    ),
+                    shlex.join(
+                        [
+                            f"POETRY_CONFIG_DIR={config_dir}",
+                            f"POETRY_CACHE_DIR={cache_dir}",
+                            f"POETRY_DATA_DIR={data_dir}",
+                            f"VIRTUAL_ENV={venv_dir}",
+                            "poetry",
+                            "install",
+                            "--no-root",
+                            "--sync",
+                        ]
+                    ),
+                ],
+                cwd=temp_dir,
+            )
+
+    def install_warm(self, requirements_file: str, *, verbose: bool) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.init(requirements_file, working_dir=temp_dir)
+
+            poetry_lock = os.path.join(temp_dir, "poetry.lock")
+            assert not os.path.exists(
+                poetry_lock
+            ), f"Lock file already exists at: {poetry_lock}"
+
+            # Run a resolution, to ensure that the lock file exists.
+            subprocess.check_call(
+                ["poetry", "lock"],
+                cwd=temp_dir,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            assert os.path.exists(
+                poetry_lock
+            ), f"Lock file doesn't exist at: {poetry_lock}"
+
+            config_dir = os.path.join(temp_dir, "config", "pypoetry")
+            cache_dir = os.path.join(temp_dir, "cache", "pypoetry")
+            data_dir = os.path.join(temp_dir, "data", "pypoetry")
+            venv_dir = os.path.join(temp_dir, ".venv")
+
+            subprocess.check_call(
+                [
+                    "hyperfine",
+                    *(["--show-output"] if verbose else []),
+                    "--command-name",
+                    f"{Tool.POETRY.value} ({Benchmark.INSTALL_WARM.value})",
+                    "--warmup",
+                    str(WARMUP),
+                    "--min-runs",
+                    str(MIN_RUNS),
+                    "--prepare",
+                    f"virtualenv --clear -p 3.10 {venv_dir} --no-seed",
+                    shlex.join(
+                        [
+                            f"POETRY_CONFIG_DIR={config_dir}",
+                            f"POETRY_CACHE_DIR={cache_dir}",
+                            f"POETRY_DATA_DIR={data_dir}",
+                            f"VIRTUAL_ENV={venv_dir}",
+                            "poetry",
+                            "install",
+                            "--no-root",
+                            "--sync",
+                        ]
+                    ),
+                ],
+                cwd=temp_dir,
+            )
+
+
 def main():
     """Run the benchmark."""
     logging.basicConfig(
@@ -376,7 +616,12 @@ def main():
         description="Benchmark Puffin against other packaging tools."
     )
     parser.add_argument(
-        "file", type=str, help="The file to read the dependencies from."
+        "file",
+        type=str,
+        help=(
+            "The file to read the dependencies from (typically: `requirements.in` "
+            "(for resolution) or `requirements.txt` (for installation))."
+        ),
     )
     parser.add_argument(
         "--verbose", "-v", action="store_true", help="Print verbose output."
@@ -413,6 +658,9 @@ def main():
         else list(Benchmark)
     )
 
+    if not os.path.exists(requirements_file):
+        raise ValueError(f"File not found: {requirements_file}")
+
     logging.info(
         "Benchmarks: {}".format(
             ", ".join([benchmark.value for benchmark in benchmarks])
@@ -434,6 +682,8 @@ def main():
                     suite = PipTools()
                 case Tool.PUFFIN:
                     suite = Puffin()
+                case Tool.POETRY:
+                    suite = Poetry()
                 case _:
                     raise ValueError(f"Invalid tool: {tool}")
 

scripts/bench/requirements.in

@@ -0,0 +1,5 @@
+pip-tools
+poetry
+tomli
+tomli_w
+virtualenv

scripts/bench/requirements.txt

@@ -0,0 +1,111 @@
+# This file was autogenerated by Puffin v0.0.1 via the following command:
+#    puffin pip-compile ./scripts/requirements.in --python-version 3.10
+build==1.0.3
+    # via
+    #   pip-tools
+    #   poetry
+cachecontrol==0.13.1
+    # via poetry
+certifi==2023.11.17
+    # via requests
+cffi==1.16.0
+    # via xattr
+charset-normalizer==3.3.2
+    # via requests
+cleo==2.1.0
+    # via poetry
+click==8.1.7
+    # via pip-tools
+crashtest==0.4.1
+    # via
+    #   cleo
+    #   poetry
+distlib==0.3.8
+    # via virtualenv
+dulwich==0.21.7
+    # via poetry
+fastjsonschema==2.19.1
+    # via poetry
+filelock==3.13.1
+    # via virtualenv
+idna==3.6
+    # via requests
+importlib-metadata==7.0.1
+    # via keyring
+installer==0.7.0
+    # via poetry
+jaraco-classes==3.3.0
+    # via keyring
+keyring==24.3.0
+    # via poetry
+more-itertools==10.1.0
+    # via jaraco-classes
+msgpack==1.0.7
+    # via cachecontrol
+packaging==23.2
+    # via
+    #   build
+    #   poetry
+pexpect==4.9.0
+    # via poetry
+pip==23.3.2
+    # via pip-tools
+pip-tools==7.3.0
+pkginfo==1.9.6
+    # via poetry
+platformdirs==3.11.0
+    # via
+    #   poetry
+    #   virtualenv
+poetry==1.7.1
+    # via poetry-plugin-export
+poetry-core==1.8.1
+    # via
+    #   poetry
+    #   poetry-plugin-export
+poetry-plugin-export==1.6.0
+    # via poetry
+ptyprocess==0.7.0
+    # via pexpect
+pycparser==2.21
+    # via cffi
+pyproject-hooks==1.0.0
+    # via
+    #   build
+    #   poetry
+rapidfuzz==3.6.1
+    # via cleo
+requests==2.31.0
+    # via
+    #   cachecontrol
+    #   poetry
+    #   requests-toolbelt
+requests-toolbelt==1.0.0
+    # via poetry
+setuptools==69.0.3
+    # via pip-tools
+shellingham==1.5.4
+    # via poetry
+tomli==2.0.1
+    # via
+    #   build
+    #   pip-tools
+    #   poetry
+    #   pyproject-hooks
+tomli-w==1.0.0
+tomlkit==0.12.3
+    # via poetry
+trove-classifiers==2023.11.29
+    # via poetry
+urllib3==2.1.0
+    # via
+    #   dulwich
+    #   requests
+virtualenv==20.25.0
+    # via poetry
+wheel==0.42.0
+    # via pip-tools
+xattr==0.10.1
+    # via poetry
+zipp==3.17.0
+    # via importlib-metadata

scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py

@@ -1,2 +1,3 @@
 from .maturin_editable import *
+
 version = 1