diff --git a/README.md b/README.md index 6aab0a7e3..49efe2a0c 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Puffin does not yet support: - VCS dependencies - URL dependencies - Uninstalling packages +- Windows - ... Like `pip-compile`, Puffin generates a platform-specific `requirements.txt` file (unlike, e.g., @@ -66,52 +67,6 @@ Options: -V, --version Print version ``` -## Benchmarks - -### Resolution - -To compare with a cold cache: - -```shell -hyperfine --runs 10 --warmup 3 --prepare "rm -f /tmp/tmp.txt" \ - "./target/release/puffin-cli compile requirements.txt --no-cache" \ - "pip-compile requirements.txt --rebuild --pip-args '--no-cache-dir' -o /tmp/tmp.txt" -``` - -To compare with a warm cache: - -```shell -hyperfine --runs 10 --warmup 3 --prepare "rm -f /tmp/tmp.txt" \ - "./target/release/puffin-cli compile requirements.txt" \ - "pip-compile requirements.txt -o /tmp/tmp.txt" -``` - -### Installation - -To compare with a cold cache: - -```shell -hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \ - "./target/release/puffin-cli sync requirements.txt --ignore-installed --no-cache" \ - "pip install -r requirements.txt --ignore-installed --no-cache-dir --no-deps" -``` - -To compare with a warm cache: - -```shell -hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \ - "./target/release/puffin-cli sync requirements.txt --ignore-installed" \ - "pip install -r requirements.txt --ignore-installed --no-deps" -``` - -To compare with all dependencies already installed: - -```shell -hyperfine --runs 10 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \ - "./target/release/puffin-cli sync requirements.txt" \ - "pip install -r requirements.txt --no-deps" -``` - ## License Puffin is licensed under either of diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 03eabff4e..000000000 --- a/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -attrs==23.1.0 - # via - # cattrs - # lsprotocol -cattrs==23.1.2 - # via lsprotocol -lsprotocol==2023.0.0b1 - # via - # -r requirements.in - # pygls -packaging==23.2 - # via -r requirements.in -pygls==1.1.1 - # via -r requirements.in -ruff==0.0.292 - # via -r requirements.in -typeguard==3.0.2 - # via pygls -typing-extensions==4.8.0 - # via -r requirements.in diff --git a/scripts/benchmarks/compile.sh b/scripts/benchmarks/compile.sh new file mode 100755 index 000000000..bc35919c3 --- /dev/null +++ b/scripts/benchmarks/compile.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env sh + +### +# Benchmark the resolver against `pip-compile`. +# +# Example usage: +# +# ./scripts/benchmarks/compile.sh ./scripts/benchmarks/requirements.in +### + +set -euxo pipefail + +TARGET=${1} + +### +# Resolution with a cold cache. +### +hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \ + "./target/release/puffin-cli compile ${TARGET} --no-cache > /tmp/requirements.txt" \ + "pip-compile ${TARGET} --rebuild --pip-args '--no-cache-dir' -o /tmp/requirements.txt" + +### +# Resolution with a warm cache. +### +hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \ + "./target/release/puffin-cli compile ${TARGET} > /tmp/requirements.txt" \ + "pip-compile ${TARGET} -o /tmp/requirements.txt" diff --git a/scripts/benchmarks/requirements-large.txt b/scripts/benchmarks/requirements-large.txt new file mode 100644 index 000000000..21be1ef3a --- /dev/null +++ b/scripts/benchmarks/requirements-large.txt @@ -0,0 +1,49 @@ +### +# A large-ish set of dependencies, including several packages with a large number of small files +# (like Django) and several packages with a small number of large files (like Ruff). +### +pygments==2.16.1 +packaging==23.2 +click==8.1.7 +threadpoolctl==3.2.0 +flake8-docstrings==1.7.0 +pytest==7.4.2 +mdurl==0.1.2 +typeguard==3.0.2 +tokenize-rt==5.2.0 +typing-extensions==4.8.0 +markupsafe==2.1.3 +attrs==23.1.0 +lsprotocol==2023.0.0b1 +markdown-it-py==3.0.0 +joblib==1.3.2 +cattrs==23.1.2 +tomlkit==0.12.1 +mccabe==0.7.0 +iniconfig==2.0.0 +rich==13.6.0 +django==5.0a1 +isort==5.12.0 +flake8==6.1.0 +snowballstemmer==2.2.0 +pycodestyle==2.11.0 +mypy-extensions==1.0.0 +pluggy==1.3.0 +pyflakes==3.1.0 +pydocstyle==6.3.0 +scipy==1.11.3 +jinja2==3.1.2 +ruff==0.0.292 +pygls==1.1.1 +pyupgrade==3.15.0 +platformdirs==3.11.0 +pylint==3.0.1 +pathspec==0.11.2 +astroid==3.0.0 +dill==0.3.7 +scikit-learn==1.3.1 +mypy==1.5.1 +numpy==1.26.0 +asgiref==3.7.2 +black==23.9.1 +sqlparse==0.4.4 diff --git a/requirements.in b/scripts/benchmarks/requirements.in similarity index 64% rename from requirements.in rename to scripts/benchmarks/requirements.in index 79d6fb80b..1a871e8d6 100644 --- a/requirements.in +++ b/scripts/benchmarks/requirements.in @@ -1,3 +1,6 @@ +### +# A small set of pure-Python packages. +### packaging>=23.1 pygls>=1.0.1 lsprotocol>=2023.0.0a1 diff --git a/scripts/benchmarks/requirements.txt b/scripts/benchmarks/requirements.txt new file mode 100644 index 000000000..3992d867b --- /dev/null +++ b/scripts/benchmarks/requirements.txt @@ -0,0 +1,10 @@ +### +# A small set of pure-Python packages. +### +attrs==23.1.0 +cattrs==23.1.2 +lsprotocol==2023.0.0b1 +packaging==23.2 +pygls==1.1.1 +typeguard==3.0.2 +typing-extensions==4.8.0 diff --git a/scripts/benchmarks/sync.sh b/scripts/benchmarks/sync.sh new file mode 100755 index 000000000..ee93ab7bd --- /dev/null +++ b/scripts/benchmarks/sync.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env sh + +### +# Benchmark the installer against `pip`. +# +# Example usage: +# +# ./scripts/benchmarks/sync.sh ./scripts/benchmarks/requirements.txt +### + +set -euxo pipefail + +TARGET=${1} + +### +# Installation with a cold cache. +### +hyperfine --runs 20 --warmup 3 \ + --prepare "rm -rf .venv && virtualenv .venv" \ + "./target/release/puffin-cli sync ${TARGET} --ignore-installed --no-cache" \ + --prepare "rm -rf /tmp/site-packages" \ + "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-cache-dir --no-deps" + +### +# Installation with a warm cache, similar to blowing away and re-creating a virtual environment. +### +hyperfine --runs 20 --warmup 3 \ + --prepare "rm -rf .venv && virtualenv .venv" \ + "./target/release/puffin-cli sync ${TARGET} --ignore-installed" \ + --prepare "rm -rf /tmp/site-packages" \ + "pip install -r ${TARGET} --target /tmp/site-packages --ignore-installed --no-deps" + +### +# Installation with all dependencies already installed (no-op). +### +hyperfine --runs 20 --warmup 3 \ + --setup "rm -rf .venv && virtualenv .venv && source .venv/bin/activate" \ + "./target/release/puffin-cli sync ${TARGET}" \ + "pip install -r ${TARGET} --no-deps"