From ee69fb51ea4b56e5f49bbd1bdb79d0be9eda561c Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 31 Jan 2024 12:31:45 -0800 Subject: [PATCH] Add PDM to benchmark script (#1214) ## Summary Overall, similar to Poetry, with some simplifications (e.g., we don't need to translate to Poetry's dependency syntax), and the need to adjust how we manage the cache and virtual environment. --- scripts/bench/__main__.py | 210 ++++++++++++++++++++++++++++++++- scripts/bench/requirements.in | 1 + scripts/bench/requirements.txt | 73 +++++++++--- 3 files changed, 263 insertions(+), 21 deletions(-) diff --git a/scripts/bench/__main__.py b/scripts/bench/__main__.py index 97c92cc95..0a7e8e5da 100644 --- a/scripts/bench/__main__.py +++ b/scripts/bench/__main__.py @@ -574,6 +574,197 @@ class Poetry(Suite): ) +class Pdm(Suite): + def __init__(self, path: str | None = None) -> None: + self.name = path or "pdm" + self.path = path or "pdm" + + def setup(self, requirements_file: str, *, cwd: str) -> None: + """Initialize a PDM project from a requirements file.""" + import tomli + import tomli_w + from packaging.requirements import Requirement + + # Parse all dependencies from the requirements file. + with open(requirements_file) as fp: + requirements = [ + Requirement(line) + for line in fp + if not line.lstrip().startswith("#") and len(line.strip()) > 0 + ] + + # Create a PDM project. + subprocess.check_call( + [self.path, "init", "--non-interactive", "--python", "3.10"], + cwd=cwd, + ) + + # Parse the pyproject.toml. + with open(os.path.join(cwd, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["project"]["dependencies"] = [ + str(requirement) for requirement in requirements + ] + + with open(os.path.join(cwd, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + def resolve_cold(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", + prepare=f"rm -rf {cache_dir} && rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--project", + cwd, + ], + ) + + def resolve_warm(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", + prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--project", + cwd, + ], + ) + + def resolve_incremental( + self, requirements_file: str, *, cwd: str + ) -> Command | None: + import tomli + import tomli_w + + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + # Add a dependency to the requirements file. + with open(os.path.join(cwd, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["project"]["dependencies"] += [INCREMENTAL_REQUIREMENT] + + with open(os.path.join(cwd, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + # Store the baseline lock file. + baseline = os.path.join(cwd, "baseline.lock") + shutil.copyfile(pdm_lock, baseline) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_INCREMENTAL.value})", + prepare=f"rm -f {pdm_lock} && cp {baseline} {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--update-reuse", + "--project", + cwd, + ], + ) + + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + venv_dir = os.path.join(cwd, ".venv") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", + prepare=( + f"rm -rf {cache_dir} && " + f"{self.path} config cache_dir {cache_dir} && " + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + ), + command=[ + f"VIRTUAL_ENV={venv_dir}", + self.path, + "sync", + "--project", + cwd, + ], + ) + + def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + venv_dir = os.path.join(cwd, ".venv") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", + prepare=( + f"{self.path} config cache_dir {cache_dir} && " + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + ), + command=[ + f"VIRTUAL_ENV={venv_dir}", + self.path, + "sync", + "--project", + cwd, + ], + ) + + class Puffin(Suite): def __init__(self, *, path: str | None = None) -> Command | None: """Initialize a Puffin benchmark.""" @@ -727,9 +918,7 @@ def main(): parser.add_argument( "--verbose", "-v", action="store_true", help="Print verbose output." ) - parser.add_argument( - "--json", action="store_true", help="Export results to JSON." - ) + parser.add_argument("--json", action="store_true", help="Export results to JSON.") parser.add_argument( "--warmup", type=int, @@ -765,6 +954,11 @@ def main(): help="Whether to benchmark Poetry (requires Poetry to be installed).", action="store_true", ) + parser.add_argument( + "--pdm", + help="Whether to benchmark PDM (requires PDM to be installed).", + action="store_true", + ) parser.add_argument( "--puffin", help="Whether to benchmark Puffin (assumes a Puffin binary exists at `./target/release/puffin`).", @@ -788,6 +982,12 @@ def main(): help="Path(s) to the Poetry binary to benchmark.", action="append", ) + parser.add_argument( + "--pdm-path", + type=str, + help="Path(s) to the PDM binary to benchmark.", + action="append", + ) parser.add_argument( "--puffin-path", type=str, @@ -819,6 +1019,8 @@ def main(): suites.append(PipCompile()) if args.poetry: suites.append(Poetry()) + if args.pdm: + suites.append(Pdm()) if args.puffin: suites.append(Puffin()) for path in args.pip_sync_path or []: @@ -827,6 +1029,8 @@ def main(): suites.append(PipCompile(path=path)) for path in args.poetry_path or []: suites.append(Poetry(path=path)) + for path in args.pdm_path or []: + suites.append(Pdm(path=path)) for path in args.puffin_path or []: suites.append(Puffin(path=path)) diff --git a/scripts/bench/requirements.in b/scripts/bench/requirements.in index b86f70e81..db0650e5c 100644 --- a/scripts/bench/requirements.in +++ b/scripts/bench/requirements.in @@ -1,3 +1,4 @@ +pdm pip-tools poetry tomli diff --git a/scripts/bench/requirements.txt b/scripts/bench/requirements.txt index a0654ce35..415c21a48 100644 --- a/scripts/bench/requirements.txt +++ b/scripts/bench/requirements.txt @@ -1,13 +1,19 @@ -# This file was autogenerated by Puffin v0.0.1 via the following command: -# puffin pip compile ./scripts/requirements.in --python-version 3.10 +# This file was autogenerated by Puffin v0.0.3 via the following command: +# puffin pip compile scripts/bench/requirements.in -o scripts/bench/requirements.txt +blinker==1.7.0 + # via pdm build==1.0.3 # via # pip-tools # poetry cachecontrol==0.13.1 - # via poetry + # via + # pdm + # poetry certifi==2023.11.17 - # via requests + # via + # pdm + # requests cffi==1.16.0 # via xattr charset-normalizer==3.3.2 @@ -20,6 +26,8 @@ crashtest==0.4.1 # via # cleo # poetry +dep-logic==0.0.4 + # via pdm distlib==0.3.8 # via virtualenv dulwich==0.21.7 @@ -27,17 +35,25 @@ dulwich==0.21.7 fastjsonschema==2.19.1 # via poetry filelock==3.13.1 - # via virtualenv + # via + # cachecontrol + # virtualenv +findpython==0.4.1 + # via pdm idna==3.6 # via requests -importlib-metadata==7.0.1 - # via keyring installer==0.7.0 - # via poetry + # via + # pdm + # poetry jaraco-classes==3.3.0 # via keyring keyring==24.3.0 # via poetry +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes msgpack==1.0.7 @@ -45,7 +61,12 @@ msgpack==1.0.7 packaging==23.2 # via # build + # dep-logic + # findpython + # pdm # poetry + # unearth +pdm==2.12.2 pexpect==4.9.0 # via poetry pip==23.3.2 @@ -55,6 +76,7 @@ pkginfo==1.9.6 # via poetry platformdirs==3.11.0 # via + # pdm # poetry # virtualenv poetry==1.7.1 @@ -69,10 +91,15 @@ ptyprocess==0.7.0 # via pexpect pycparser==2.21 # via cffi +pygments==2.17.2 + # via rich pyproject-hooks==1.0.0 # via # build + # pdm # poetry +python-dotenv==1.0.1 + # via pdm rapidfuzz==3.6.1 # via cleo requests==2.31.0 @@ -80,32 +107,42 @@ requests==2.31.0 # cachecontrol # poetry # requests-toolbelt + # unearth requests-toolbelt==1.0.0 - # via poetry + # via + # pdm + # poetry +resolvelib==1.0.1 + # via pdm +rich==13.7.0 + # via pdm setuptools==69.0.3 # via pip-tools shellingham==1.5.4 - # via poetry -tomli==2.0.1 # via - # build - # pip-tools + # pdm # poetry - # pyproject-hooks +tomli==2.0.1 tomli-w==1.0.0 tomlkit==0.12.3 - # via poetry + # via + # pdm + # poetry trove-classifiers==2023.11.29 # via poetry +truststore==0.8.0 + # via pdm +unearth==0.14.0 + # via pdm urllib3==2.1.0 # via # dulwich # requests virtualenv==20.25.0 - # via poetry + # via + # pdm + # poetry wheel==0.42.0 # via pip-tools xattr==0.10.1 # via poetry -zipp==3.17.0 - # via importlib-metadata