From 5b7e8bcd721127740b8c7178e409fca603c3206b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sun, 14 Dec 2025 18:24:23 -0800 Subject: [PATCH] bench: offline benchmarks (#1444) --- .github/workflows/benchmark.yml | 2 - bench/test_bench_offline.py | 68 +++++++++++++++++++++++++++++++++ docs/development.md | 21 ++++------ pyproject.toml | 1 + uv.lock | 11 ++++++ 5 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 bench/test_bench_offline.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d2d49db1..403e19d1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,5 +49,3 @@ jobs: with: mode: walltime run: make bench - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/bench/test_bench_offline.py b/bench/test_bench_offline.py new file mode 100644 index 00000000..32952c60 --- /dev/null +++ b/bench/test_bench_offline.py @@ -0,0 +1,68 @@ +import io +import zipfile +from pathlib import Path + +import pytest +import urllib3 + +from bench.common import zizmor + + +@pytest.fixture(scope="session") +def grafana(tmp_path_factory) -> Path: + archive = "https://github.com/grafana/grafana/archive/9f212d11d0ac9c38ada62a7db830844bb9b02905.zip" + raw_zip = urllib3.PoolManager().request("GET", archive).data + + path = tmp_path_factory.mktemp("grafana") + + zipfile.ZipFile(io.BytesIO(raw_zip)).extractall(path) + + return path + + +@pytest.fixture(scope="session") +def cpython(tmp_path_factory) -> Path: + archive = "https://github.com/python/cpython/archive/48f88310044c6ef877f3b0761cf7afece2f8fb3a.zip" + raw_zip = urllib3.PoolManager().request("GET", archive).data + + path = tmp_path_factory.mktemp("cpython") + + zipfile.ZipFile(io.BytesIO(raw_zip)).extractall(path) + + return path + + +@pytest.mark.benchmark +def test_zizmor_offline_grafana_9f212d11d0(grafana: Path): + """ + Runs `zizmor --offline --format=plain --no-exit-codes --no-config ` + """ + + zizmor( + [ + "--offline", + "--format=plain", + "--no-exit-codes", + "--no-config", + str(grafana), + ], + check=True, + ) + + +@pytest.mark.benchmark +def test_zizmor_offline_cpython_48f88310044c(cpython: Path): + """ + Runs `zizmor --offline --format=plain --no-exit-codes --no-config ` + """ + + zizmor( + [ + "--offline", + "--format=plain", + "--no-exit-codes", + "--no-config", + str(cpython), + ], + check=True, + ) diff --git a/docs/development.md b/docs/development.md index db5e4448..0f3bd128 100644 --- a/docs/development.md +++ b/docs/development.md @@ -172,7 +172,7 @@ See [insta's documentation] for more details. ## Benchmarking -`zizmor` currently uses [hyperfine](https://github.org/sharkdp/hyperfine) +`zizmor` currently uses [pytest-codspeed](https://github.com/CodSpeedHQ/pytest-codspeed) for command-line benchmarking. Benchmarks are stored in the top-level `bench/` directory, and can be @@ -184,27 +184,22 @@ make bench ``` We currently run offline benchmarks in the CI and report their results -to [Bencher](https://bencher.dev/). See -[our project page](https://bencher.dev/console/projects/zizmor/plots) -on Bencher for results and trends. +to [CodSpeed](https://codspeed.io). See +[our project page](https://codspeed.io/zizmorcore/zizmor) +on CodSpeed for results and trends. There are also online benchmarks, but these don't get run automatically. -To run them, you can pass `GH_TOKEN` to the `bench/benchmark.py` script -directly: +To run them, you can set `GH_TOKEN`: ```bash -GH_TOKEN=$(gh auth token) uv run bench/benchmark.py +GH_TOKEN=$(gh auth token) make bench ``` ### Adding new benchmarks -`zizmor` currently orchestrates benchmarks with `bench/benchmark.py`, -which wraps `hyperfine` to add a planning phase. +Benchmarks are currently written as pytest functions. -Take a look at `bench/benchmarks.json` for the current benchmarks. -Observe that each benchmark tells `benchmark.py` how to retrieve its -input as well as provides a `stencil` that the benchmark runner will -expand to run the benchmark. +Take a look at `bench/test_*.py` for existing benchmarks. ## Building the website diff --git a/pyproject.toml b/pyproject.toml index 33b5eada..a9de525a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,5 +19,6 @@ include = [{ path = "README.md", format = "sdist" }, { path = "LICENSE", format bench = [ "pytest>=9.0.2", "pytest-codspeed>=4.2.0", + "urllib3>=2.6.2", ] docs = ["zensical"] diff --git a/uv.lock b/uv.lock index af28567c..cc92b8e7 100644 --- a/uv.lock +++ b/uv.lock @@ -393,6 +393,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "urllib3" +version = "2.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/24/a2a2ed9addd907787d7aa0355ba36a6cadf1768b934c652ea78acbd59dcd/urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797", size = 432930, upload-time = "2025-12-11T15:56:40.252Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/b9/4095b668ea3678bf6a0af005527f39de12fb026516fb3df17495a733b7f8/urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd", size = 131182, upload-time = "2025-12-11T15:56:38.584Z" }, +] + [[package]] name = "zensical" version = "0.0.10" @@ -430,6 +439,7 @@ source = { editable = "." } bench = [ { name = "pytest" }, { name = "pytest-codspeed" }, + { name = "urllib3" }, ] docs = [ { name = "zensical" }, @@ -441,5 +451,6 @@ docs = [ bench = [ { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-codspeed", specifier = ">=4.2.0" }, + { name = "urllib3", specifier = ">=2.6.2" }, ] docs = [{ name = "zensical" }]