Add benchmark scripts (#69)

Moving these out of the README and into proper scripts.

scripts/benchmarks/compile.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+###
+# Benchmark the resolver against `pip-compile`.
+#
+# Example usage:
+#
+#   ./scripts/benchmarks/compile.sh ./scripts/benchmarks/requirements.in
+###
+
+set -euxo pipefail
+
+TARGET=${1}
+
+###
+# Resolution with a cold cache.
+###
+hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \
+    "./target/release/puffin-cli compile ${TARGET} --no-cache > /tmp/requirements.txt" \
+    "pip-compile ${TARGET} --rebuild --pip-args '--no-cache-dir' -o /tmp/requirements.txt"
+
+###
+# Resolution with a warm cache.
+###
+hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \
+    "./target/release/puffin-cli compile ${TARGET} > /tmp/requirements.txt" \
+    "pip-compile ${TARGET} -o /tmp/requirements.txt"

scripts/benchmarks/requirements-large.txt

@@ -0,0 +1,49 @@
+###
+# A large-ish set of dependencies, including several packages with a large number of small files
+# (like Django) and several packages with a small number of large files (like Ruff).
+###
+pygments==2.16.1
+packaging==23.2
+click==8.1.7
+threadpoolctl==3.2.0
+flake8-docstrings==1.7.0
+pytest==7.4.2
+mdurl==0.1.2
+typeguard==3.0.2
+tokenize-rt==5.2.0
+typing-extensions==4.8.0
+markupsafe==2.1.3
+attrs==23.1.0
+lsprotocol==2023.0.0b1
+markdown-it-py==3.0.0
+joblib==1.3.2
+cattrs==23.1.2
+tomlkit==0.12.1
+mccabe==0.7.0
+iniconfig==2.0.0
+rich==13.6.0
+django==5.0a1
+isort==5.12.0
+flake8==6.1.0
+snowballstemmer==2.2.0
+pycodestyle==2.11.0
+mypy-extensions==1.0.0
+pluggy==1.3.0
+pyflakes==3.1.0
+pydocstyle==6.3.0
+scipy==1.11.3
+jinja2==3.1.2
+ruff==0.0.292
+pygls==1.1.1
+pyupgrade==3.15.0
+platformdirs==3.11.0
+pylint==3.0.1
+pathspec==0.11.2
+astroid==3.0.0
+dill==0.3.7
+scikit-learn==1.3.1
+mypy==1.5.1
+numpy==1.26.0
+asgiref==3.7.2
+black==23.9.1
+sqlparse==0.4.4

scripts/benchmarks/requirements.in

@@ -0,0 +1,8 @@
+###
+# A small set of pure-Python packages.
+###
+packaging>=23.1
+pygls>=1.0.1
+lsprotocol>=2023.0.0a1
+ruff>=0.0.274
+typing_extensions

scripts/benchmarks/requirements.txt

@@ -0,0 +1,10 @@
+###
+# A small set of pure-Python packages.
+###
+attrs==23.1.0
+cattrs==23.1.2
+lsprotocol==2023.0.0b1
+packaging==23.2
+pygls==1.1.1
+typeguard==3.0.2
+typing-extensions==4.8.0

scripts/benchmarks/sync.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+
+###
+# Benchmark the installer against `pip`.
+#
+# Example usage:
+#
+#   ./scripts/benchmarks/sync.sh ./scripts/benchmarks/requirements.txt
+###
+
+set -euxo pipefail
+
+TARGET=${1}
+
+###
+# Installation with a cold cache.
+###
+hyperfine --runs 20 --warmup 3 \
+    --prepare "rm -rf .venv && virtualenv .venv" \
+    "./target/release/puffin-cli sync ${TARGET} --ignore-installed --no-cache" \
+    --prepare "rm -rf /tmp/site-packages" \
+    "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-cache-dir --no-deps"
+
+###
+# Installation with a warm cache, similar to blowing away and re-creating a virtual environment.
+###
+hyperfine --runs 20 --warmup 3 \
+    --prepare "rm -rf .venv && virtualenv .venv" \
+    "./target/release/puffin-cli sync ${TARGET} --ignore-installed" \
+    --prepare "rm -rf /tmp/site-packages" \
+    "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-deps"
+
+###
+# Installation with all dependencies already installed (no-op).
+###
+hyperfine --runs 20 --warmup 3 \
+    --setup "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \
+    "./target/release/puffin-cli sync ${TARGET}" \
+    "pip install -r ${TARGET} --no-deps"