From d2d87db7a3d149328f8313e44f10e2bea016fb64 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 5 Jan 2024 21:52:55 -0500 Subject: [PATCH] Add Poetry support to `bench.py` (#803) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Enables benchmarking against Poetry for resolution and installation: ``` Benchmark 1: pip-tools (resolve-cold) Time (mean ± σ): 962.7 ms ± 241.9 ms [User: 322.8 ms, System: 80.5 ms] Range (min … max): 714.9 ms … 1459.4 ms 10 runs Benchmark 1: puffin (resolve-cold) Time (mean ± σ): 193.2 ms ± 8.2 ms [User: 31.3 ms, System: 22.8 ms] Range (min … max): 179.8 ms … 206.4 ms 14 runs Benchmark 1: poetry (resolve-cold) Time (mean ± σ): 900.7 ms ± 21.2 ms [User: 371.6 ms, System: 92.1 ms] Range (min … max): 855.7 ms … 933.4 ms 10 runs Benchmark 1: pip-tools (resolve-warm) Time (mean ± σ): 386.0 ms ± 19.1 ms [User: 255.8 ms, System: 46.2 ms] Range (min … max): 368.7 ms … 434.5 ms 10 runs Benchmark 1: puffin (resolve-warm) Time (mean ± σ): 8.1 ms ± 0.4 ms [User: 4.4 ms, System: 5.1 ms] Range (min … max): 7.5 ms … 11.1 ms 183 runs Benchmark 1: poetry (resolve-warm) Time (mean ± σ): 336.3 ms ± 0.6 ms [User: 283.6 ms, System: 44.7 ms] Range (min … max): 335.0 ms … 337.3 ms 10 runs ``` --- scripts/{bench.py => bench/__main__.py} | 256 +++++++++++++++++- scripts/bench/requirements.in | 5 + scripts/bench/requirements.txt | 111 ++++++++ .../python/maturin_editable/__init__.py | 1 + 4 files changed, 370 insertions(+), 3 deletions(-) rename scripts/{bench.py => bench/__main__.py} (62%) create mode 100644 scripts/bench/requirements.in create mode 100644 scripts/bench/requirements.txt diff --git a/scripts/bench.py b/scripts/bench/__main__.py similarity index 62% rename from scripts/bench.py rename to scripts/bench/__main__.py index ecb28c104..3dd48b82e 100644 --- a/scripts/bench.py +++ b/scripts/bench/__main__.py @@ -1,10 +1,20 @@ """Benchmark Puffin against other packaging tools. -This script assumes that `pip`, `pip-tools`, `virtualenv`, and `hyperfine` are +This script assumes that `pip`, `pip-tools`, `virtualenv`, `poetry` and `hyperfine` are installed, and that a Puffin release builds exists at `./target/release/puffin` (relative to the repository root). -Example usage: python bench.py -t puffin -t pip-tools requirements.in +This script assumes that Python 3.10 is installed. + +To set up the required environment, run: + + cargo build --release + ./target/release/puffin venv + ./target/release/puffin pip-sync ./scripts/requirements.txt + +Example usage: + + python -m scripts.bench -t puffin -t pip-tools requirements.in """ import abc import argparse @@ -15,6 +25,10 @@ import shlex import subprocess import tempfile +import tomli +import tomli_w +from packaging.requirements import Requirement + WARMUP = 3 MIN_RUNS = 10 @@ -24,6 +38,7 @@ class Tool(enum.Enum): PIP_TOOLS = "pip-tools" PUFFIN = "puffin" + POETRY = "poetry" class Benchmark(enum.Enum): @@ -364,6 +379,231 @@ class Puffin(Suite): ) +class Poetry(Suite): + def init(self, requirements_file: str, *, working_dir: str) -> None: + """Initialize a Poetry project from a requirements file.""" + # Parse all dependencies from the requirements file. + with open(requirements_file) as fp: + requirements = [ + Requirement(line) for line in fp if not line.startswith("#") + ] + + # Create a Poetry project. + subprocess.check_call( + [ + "poetry", + "init", + "--name", + "bench", + "--no-interaction", + "--python", + ">=3.10", + ], + cwd=working_dir, + ) + + # Parse the pyproject.toml. + with open(os.path.join(working_dir, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["tool"]["poetry"]["dependencies"].update( + { + str(requirement.name): str(requirement.specifier) + if requirement.specifier + else "*" + for requirement in requirements + } + ) + + with open(os.path.join(working_dir, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + def resolve_cold(self, requirements_file: str, *, verbose: bool) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + self.init(requirements_file, working_dir=temp_dir) + + poetry_lock = os.path.join(temp_dir, "poetry.lock") + config_dir = os.path.join(temp_dir, "config", "pypoetry") + cache_dir = os.path.join(temp_dir, "cache", "pypoetry") + data_dir = os.path.join(temp_dir, "data", "pypoetry") + + subprocess.check_call( + [ + "hyperfine", + *(["--show-output"] if verbose else []), + "--command-name", + f"{Tool.POETRY.value} ({Benchmark.RESOLVE_COLD.value})", + "--warmup", + str(WARMUP), + "--min-runs", + str(MIN_RUNS), + "--prepare", + ( + f"rm -rf {config_dir} && " + f"rm -rf {cache_dir} && " + f"rm -rf {data_dir} &&" + f"rm -rf {poetry_lock}" + ), + shlex.join( + [ + f"POETRY_CONFIG_DIR={config_dir}", + f"POETRY_CACHE_DIR={cache_dir}", + f"POETRY_DATA_DIR={data_dir}", + "poetry", + "lock", + ] + ), + ], + cwd=temp_dir, + ) + + def resolve_warm(self, requirements_file: str, *, verbose: bool) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + self.init(requirements_file, working_dir=temp_dir) + + poetry_lock = os.path.join(temp_dir, "poetry.lock") + config_dir = os.path.join(temp_dir, "config", "pypoetry") + cache_dir = os.path.join(temp_dir, "cache", "pypoetry") + data_dir = os.path.join(temp_dir, "data", "pypoetry") + + subprocess.check_call( + [ + "hyperfine", + *(["--show-output"] if verbose else []), + "--command-name", + f"{Tool.POETRY.value} ({Benchmark.RESOLVE_WARM.value})", + "--warmup", + str(WARMUP), + "--min-runs", + str(MIN_RUNS), + "--prepare", + f"rm -rf {poetry_lock}", + shlex.join( + [ + f"POETRY_CONFIG_DIR={config_dir}", + f"POETRY_CACHE_DIR={cache_dir}", + f"POETRY_DATA_DIR={data_dir}", + "poetry", + "lock", + ] + ), + ], + cwd=temp_dir, + ) + + def install_cold(self, requirements_file: str, *, verbose: bool) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + self.init(requirements_file, working_dir=temp_dir) + + poetry_lock = os.path.join(temp_dir, "poetry.lock") + assert not os.path.exists( + poetry_lock + ), f"Lock file already exists at: {poetry_lock}" + + # Run a resolution, to ensure that the lock file exists. + subprocess.check_call( + ["poetry", "lock"], + cwd=temp_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists( + poetry_lock + ), f"Lock file doesn't exist at: {poetry_lock}" + + config_dir = os.path.join(temp_dir, "config", "pypoetry") + cache_dir = os.path.join(temp_dir, "cache", "pypoetry") + data_dir = os.path.join(temp_dir, "data", "pypoetry") + venv_dir = os.path.join(temp_dir, ".venv") + + subprocess.check_call( + [ + "hyperfine", + *(["--show-output"] if verbose else []), + "--command-name", + f"{Tool.POETRY.value} ({Benchmark.INSTALL_COLD.value})", + "--warmup", + str(WARMUP), + "--min-runs", + str(MIN_RUNS), + "--prepare", + ( + f"rm -rf {config_dir} && " + f"rm -rf {cache_dir} && " + f"rm -rf {data_dir} &&" + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + ), + shlex.join( + [ + f"POETRY_CONFIG_DIR={config_dir}", + f"POETRY_CACHE_DIR={cache_dir}", + f"POETRY_DATA_DIR={data_dir}", + f"VIRTUAL_ENV={venv_dir}", + "poetry", + "install", + "--no-root", + "--sync", + ] + ), + ], + cwd=temp_dir, + ) + + def install_warm(self, requirements_file: str, *, verbose: bool) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + self.init(requirements_file, working_dir=temp_dir) + + poetry_lock = os.path.join(temp_dir, "poetry.lock") + assert not os.path.exists( + poetry_lock + ), f"Lock file already exists at: {poetry_lock}" + + # Run a resolution, to ensure that the lock file exists. + subprocess.check_call( + ["poetry", "lock"], + cwd=temp_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists( + poetry_lock + ), f"Lock file doesn't exist at: {poetry_lock}" + + config_dir = os.path.join(temp_dir, "config", "pypoetry") + cache_dir = os.path.join(temp_dir, "cache", "pypoetry") + data_dir = os.path.join(temp_dir, "data", "pypoetry") + venv_dir = os.path.join(temp_dir, ".venv") + + subprocess.check_call( + [ + "hyperfine", + *(["--show-output"] if verbose else []), + "--command-name", + f"{Tool.POETRY.value} ({Benchmark.INSTALL_WARM.value})", + "--warmup", + str(WARMUP), + "--min-runs", + str(MIN_RUNS), + "--prepare", + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed", + shlex.join( + [ + f"POETRY_CONFIG_DIR={config_dir}", + f"POETRY_CACHE_DIR={cache_dir}", + f"POETRY_DATA_DIR={data_dir}", + f"VIRTUAL_ENV={venv_dir}", + "poetry", + "install", + "--no-root", + "--sync", + ] + ), + ], + cwd=temp_dir, + ) + + def main(): """Run the benchmark.""" logging.basicConfig( @@ -376,7 +616,12 @@ def main(): description="Benchmark Puffin against other packaging tools." ) parser.add_argument( - "file", type=str, help="The file to read the dependencies from." + "file", + type=str, + help=( + "The file to read the dependencies from (typically: `requirements.in` " + "(for resolution) or `requirements.txt` (for installation))." + ), ) parser.add_argument( "--verbose", "-v", action="store_true", help="Print verbose output." @@ -413,6 +658,9 @@ def main(): else list(Benchmark) ) + if not os.path.exists(requirements_file): + raise ValueError(f"File not found: {requirements_file}") + logging.info( "Benchmarks: {}".format( ", ".join([benchmark.value for benchmark in benchmarks]) @@ -434,6 +682,8 @@ def main(): suite = PipTools() case Tool.PUFFIN: suite = Puffin() + case Tool.POETRY: + suite = Poetry() case _: raise ValueError(f"Invalid tool: {tool}") diff --git a/scripts/bench/requirements.in b/scripts/bench/requirements.in new file mode 100644 index 000000000..b86f70e81 --- /dev/null +++ b/scripts/bench/requirements.in @@ -0,0 +1,5 @@ +pip-tools +poetry +tomli +tomli_w +virtualenv diff --git a/scripts/bench/requirements.txt b/scripts/bench/requirements.txt new file mode 100644 index 000000000..d15598762 --- /dev/null +++ b/scripts/bench/requirements.txt @@ -0,0 +1,111 @@ +# This file was autogenerated by Puffin v0.0.1 via the following command: +# puffin pip-compile ./scripts/requirements.in --python-version 3.10 +build==1.0.3 + # via + # pip-tools + # poetry +cachecontrol==0.13.1 + # via poetry +certifi==2023.11.17 + # via requests +cffi==1.16.0 + # via xattr +charset-normalizer==3.3.2 + # via requests +cleo==2.1.0 + # via poetry +click==8.1.7 + # via pip-tools +crashtest==0.4.1 + # via + # cleo + # poetry +distlib==0.3.8 + # via virtualenv +dulwich==0.21.7 + # via poetry +fastjsonschema==2.19.1 + # via poetry +filelock==3.13.1 + # via virtualenv +idna==3.6 + # via requests +importlib-metadata==7.0.1 + # via keyring +installer==0.7.0 + # via poetry +jaraco-classes==3.3.0 + # via keyring +keyring==24.3.0 + # via poetry +more-itertools==10.1.0 + # via jaraco-classes +msgpack==1.0.7 + # via cachecontrol +packaging==23.2 + # via + # build + # poetry +pexpect==4.9.0 + # via poetry +pip==23.3.2 + # via pip-tools +pip-tools==7.3.0 +pkginfo==1.9.6 + # via poetry +platformdirs==3.11.0 + # via + # poetry + # virtualenv +poetry==1.7.1 + # via poetry-plugin-export +poetry-core==1.8.1 + # via + # poetry + # poetry-plugin-export +poetry-plugin-export==1.6.0 + # via poetry +ptyprocess==0.7.0 + # via pexpect +pycparser==2.21 + # via cffi +pyproject-hooks==1.0.0 + # via + # build + # poetry +rapidfuzz==3.6.1 + # via cleo +requests==2.31.0 + # via + # cachecontrol + # poetry + # requests-toolbelt +requests-toolbelt==1.0.0 + # via poetry +setuptools==69.0.3 + # via pip-tools +shellingham==1.5.4 + # via poetry +tomli==2.0.1 + # via + # build + # pip-tools + # poetry + # pyproject-hooks +tomli-w==1.0.0 +tomlkit==0.12.3 + # via poetry +trove-classifiers==2023.11.29 + # via poetry +urllib3==2.1.0 + # via + # dulwich + # requests +virtualenv==20.25.0 + # via poetry +wheel==0.42.0 + # via pip-tools +xattr==0.10.1 + # via poetry +zipp==3.17.0 + # via importlib-metadata diff --git a/scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py b/scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py index 5f7a2bd82..2712f2bc6 100644 --- a/scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py +++ b/scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py @@ -1,2 +1,3 @@ from .maturin_editable import * + version = 1