Add benchmark scripts (#69)

Moving these out of the README and into proper scripts.

README.md

@@ -29,6 +29,7 @@ Puffin does not yet support:
 - VCS dependencies
 - URL dependencies
 - Uninstalling packages
+- Windows
 - ...
 
 Like `pip-compile`, Puffin generates a platform-specific `requirements.txt` file (unlike, e.g.,
@@ -66,52 +67,6 @@ Options:
   -V, --version  Print version
 ```
 
-## Benchmarks
-
-### Resolution
-
-To compare with a cold cache:
-
-```shell
-hyperfine --runs 10 --warmup 3 --prepare "rm -f /tmp/tmp.txt" \
-    "./target/release/puffin-cli compile requirements.txt --no-cache" \
-    "pip-compile requirements.txt --rebuild --pip-args '--no-cache-dir' -o /tmp/tmp.txt"
-```
-
-To compare with a warm cache:
-
-```shell
-hyperfine --runs 10 --warmup 3 --prepare "rm -f /tmp/tmp.txt" \
-    "./target/release/puffin-cli compile requirements.txt" \
-    "pip-compile requirements.txt -o /tmp/tmp.txt"
-```
-
-### Installation
-
-To compare with a cold cache:
-
-```shell
-hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
-    "./target/release/puffin-cli sync requirements.txt --ignore-installed --no-cache" \
-    "pip install -r requirements.txt --ignore-installed --no-cache-dir --no-deps"
-```
-
-To compare with a warm cache:
-
-```shell
-hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
-    "./target/release/puffin-cli sync requirements.txt --ignore-installed" \
-    "pip install -r requirements.txt --ignore-installed --no-deps"
-```
-
-To compare with all dependencies already installed:
-
-```shell
-hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
-    "./target/release/puffin-cli sync requirements.txt" \
-    "pip install -r requirements.txt --no-deps"
-```
-
 ## License
 
 Puffin is licensed under either of

requirements.txt

@@ -1,26 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile requirements.in
-#
-attrs==23.1.0
-    # via
-    #   cattrs
-    #   lsprotocol
-cattrs==23.1.2
-    # via lsprotocol
-lsprotocol==2023.0.0b1
-    # via
-    #   -r requirements.in
-    #   pygls
-packaging==23.2
-    # via -r requirements.in
-pygls==1.1.1
-    # via -r requirements.in
-ruff==0.0.292
-    # via -r requirements.in
-typeguard==3.0.2
-    # via pygls
-typing-extensions==4.8.0
-    # via -r requirements.in

scripts/benchmarks/compile.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+###
+# Benchmark the resolver against `pip-compile`.
+#
+# Example usage:
+#
+#   ./scripts/benchmarks/compile.sh ./scripts/benchmarks/requirements.in
+###
+
+set -euxo pipefail
+
+TARGET=${1}
+
+###
+# Resolution with a cold cache.
+###
+hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \
+    "./target/release/puffin-cli compile ${TARGET} --no-cache > /tmp/requirements.txt" \
+    "pip-compile ${TARGET} --rebuild --pip-args '--no-cache-dir' -o /tmp/requirements.txt"
+
+###
+# Resolution with a warm cache.
+###
+hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \
+    "./target/release/puffin-cli compile ${TARGET} > /tmp/requirements.txt" \
+    "pip-compile ${TARGET} -o /tmp/requirements.txt"

scripts/benchmarks/requirements-large.txt

@@ -0,0 +1,49 @@
+###
+# A large-ish set of dependencies, including several packages with a large number of small files
+# (like Django) and several packages with a small number of large files (like Ruff).
+###
+pygments==2.16.1
+packaging==23.2
+click==8.1.7
+threadpoolctl==3.2.0
+flake8-docstrings==1.7.0
+pytest==7.4.2
+mdurl==0.1.2
+typeguard==3.0.2
+tokenize-rt==5.2.0
+typing-extensions==4.8.0
+markupsafe==2.1.3
+attrs==23.1.0
+lsprotocol==2023.0.0b1
+markdown-it-py==3.0.0
+joblib==1.3.2
+cattrs==23.1.2
+tomlkit==0.12.1
+mccabe==0.7.0
+iniconfig==2.0.0
+rich==13.6.0
+django==5.0a1
+isort==5.12.0
+flake8==6.1.0
+snowballstemmer==2.2.0
+pycodestyle==2.11.0
+mypy-extensions==1.0.0
+pluggy==1.3.0
+pyflakes==3.1.0
+pydocstyle==6.3.0
+scipy==1.11.3
+jinja2==3.1.2
+ruff==0.0.292
+pygls==1.1.1
+pyupgrade==3.15.0
+platformdirs==3.11.0
+pylint==3.0.1
+pathspec==0.11.2
+astroid==3.0.0
+dill==0.3.7
+scikit-learn==1.3.1
+mypy==1.5.1
+numpy==1.26.0
+asgiref==3.7.2
+black==23.9.1
+sqlparse==0.4.4

scripts/benchmarks/requirements.in

@@ -1,3 +1,6 @@
+###
+# A small set of pure-Python packages.
+###
 packaging>=23.1
 pygls>=1.0.1
 lsprotocol>=2023.0.0a1

scripts/benchmarks/requirements.txt

@@ -0,0 +1,10 @@
+###
+# A small set of pure-Python packages.
+###
+attrs==23.1.0
+cattrs==23.1.2
+lsprotocol==2023.0.0b1
+packaging==23.2
+pygls==1.1.1
+typeguard==3.0.2
+typing-extensions==4.8.0

scripts/benchmarks/sync.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+
+###
+# Benchmark the installer against `pip`.
+#
+# Example usage:
+#
+#   ./scripts/benchmarks/sync.sh ./scripts/benchmarks/requirements.txt
+###
+
+set -euxo pipefail
+
+TARGET=${1}
+
+###
+# Installation with a cold cache.
+###
+hyperfine --runs 20 --warmup 3 \
+    --prepare "rm -rf .venv && virtualenv .venv" \
+    "./target/release/puffin-cli sync ${TARGET} --ignore-installed --no-cache" \
+    --prepare "rm -rf /tmp/site-packages" \
+    "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-cache-dir --no-deps"
+
+###
+# Installation with a warm cache, similar to blowing away and re-creating a virtual environment.
+###
+hyperfine --runs 20 --warmup 3 \
+    --prepare "rm -rf .venv && virtualenv .venv" \
+    "./target/release/puffin-cli sync ${TARGET} --ignore-installed" \
+    --prepare "rm -rf /tmp/site-packages" \
+    "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-deps"
+
+###
+# Installation with all dependencies already installed (no-op).
+###
+hyperfine --runs 20 --warmup 3 \
+    --setup "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
+    "./target/release/puffin-cli sync ${TARGET}" \
+    "pip install -r ${TARGET} --no-deps"