bench: offline benchmarks (#1444)

.github/workflows/benchmark.yml

@@ -49,5 +49,3 @@ jobs:
         with:
           mode: walltime
           run: make bench
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

bench/test_bench_offline.py

@@ -0,0 +1,68 @@
+import io
+import zipfile
+from pathlib import Path
+
+import pytest
+import urllib3
+
+from bench.common import zizmor
+
+
+@pytest.fixture(scope="session")
+def grafana(tmp_path_factory) -> Path:
+    archive = "https://github.com/grafana/grafana/archive/9f212d11d0ac9c38ada62a7db830844bb9b02905.zip"
+    raw_zip = urllib3.PoolManager().request("GET", archive).data
+
+    path = tmp_path_factory.mktemp("grafana")
+
+    zipfile.ZipFile(io.BytesIO(raw_zip)).extractall(path)
+
+    return path
+
+
+@pytest.fixture(scope="session")
+def cpython(tmp_path_factory) -> Path:
+    archive = "https://github.com/python/cpython/archive/48f88310044c6ef877f3b0761cf7afece2f8fb3a.zip"
+    raw_zip = urllib3.PoolManager().request("GET", archive).data
+
+    path = tmp_path_factory.mktemp("cpython")
+
+    zipfile.ZipFile(io.BytesIO(raw_zip)).extractall(path)
+
+    return path
+
+
+@pytest.mark.benchmark
+def test_zizmor_offline_grafana_9f212d11d0(grafana: Path):
+    """
+    Runs `zizmor --offline --format=plain --no-exit-codes --no-config <path-to-grafana-source>`
+    """
+
+    zizmor(
+        [
+            "--offline",
+            "--format=plain",
+            "--no-exit-codes",
+            "--no-config",
+            str(grafana),
+        ],
+        check=True,
+    )
+
+
+@pytest.mark.benchmark
+def test_zizmor_offline_cpython_48f88310044c(cpython: Path):
+    """
+    Runs `zizmor --offline --format=plain --no-exit-codes --no-config <path-to-cpython-source>`
+    """
+
+    zizmor(
+        [
+            "--offline",
+            "--format=plain",
+            "--no-exit-codes",
+            "--no-config",
+            str(cpython),
+        ],
+        check=True,
+    )

docs/development.md

@@ -172,7 +172,7 @@ See [insta's documentation] for more details.
 
 ## Benchmarking
 
-`zizmor` currently uses [hyperfine](https://github.org/sharkdp/hyperfine)
+`zizmor` currently uses [pytest-codspeed](https://github.com/CodSpeedHQ/pytest-codspeed)
 for command-line benchmarking.
 
 Benchmarks are stored in the top-level `bench/` directory, and can be
@@ -184,27 +184,22 @@ make bench
 ```
 
 We currently run offline benchmarks in the CI and report their results
-to [Bencher](https://bencher.dev/). See
-[our project page](https://bencher.dev/console/projects/zizmor/plots)
-on Bencher for results and trends.
+to [CodSpeed](https://codspeed.io). See
+[our project page](https://codspeed.io/zizmorcore/zizmor)
+on CodSpeed for results and trends.
 
 There are also online benchmarks, but these don't get run automatically.
-To run them, you can pass `GH_TOKEN` to the `bench/benchmark.py` script
-directly:
+To run them, you can set `GH_TOKEN`:
 
 ```bash
-GH_TOKEN=$(gh auth token) uv run bench/benchmark.py
+GH_TOKEN=$(gh auth token) make bench
 ```
 
 ### Adding new benchmarks
 
-`zizmor` currently orchestrates benchmarks with `bench/benchmark.py`,
-which wraps `hyperfine` to add a planning phase.
+Benchmarks are currently written as pytest functions.
 
-Take a look at `bench/benchmarks.json` for the current benchmarks.
-Observe that each benchmark tells `benchmark.py` how to retrieve its
-input as well as provides a `stencil` that the benchmark runner will
-expand to run the benchmark.
+Take a look at `bench/test_*.py` for existing benchmarks.
 
 ## Building the website
 

pyproject.toml

@@ -19,5 +19,6 @@ include = [{ path = "README.md", format = "sdist" }, { path = "LICENSE", format
 bench = [
     "pytest>=9.0.2",
     "pytest-codspeed>=4.2.0",
+    "urllib3>=2.6.2",
 ]
 docs = ["zensical"]

uv.lock

@@ -393,6 +393,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
 ]
 
+[[package]]
+name = "urllib3"
+version = "2.6.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1e/24/a2a2ed9addd907787d7aa0355ba36a6cadf1768b934c652ea78acbd59dcd/urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797", size = 432930, upload-time = "2025-12-11T15:56:40.252Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6d/b9/4095b668ea3678bf6a0af005527f39de12fb026516fb3df17495a733b7f8/urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd", size = 131182, upload-time = "2025-12-11T15:56:38.584Z" },
+]
+
 [[package]]
 name = "zensical"
 version = "0.0.10"
@@ -430,6 +439,7 @@ source = { editable = "." }
 bench = [
     { name = "pytest" },
     { name = "pytest-codspeed" },
+    { name = "urllib3" },
 ]
 docs = [
     { name = "zensical" },
@@ -441,5 +451,6 @@ docs = [
 bench = [
     { name = "pytest", specifier = ">=9.0.2" },
     { name = "pytest-codspeed", specifier = ">=4.2.0" },
+    { name = "urllib3", specifier = ">=2.6.2" },
 ]
 docs = [{ name = "zensical" }]