bench: add support for online benchmarks (#1258)

Makefile

@@ -49,4 +49,4 @@ pinact:
 
 .PHONY: bench
 bench:
-	uv run bench/benchmark.py
+	uv run bench/benchmark.py --offline

bench/benchmark.py

@@ -5,6 +5,7 @@
 import argparse
 import hashlib
 import json
+import os
 import shlex
 import shutil
 import subprocess
@@ -31,6 +32,8 @@ _RESULTS.mkdir(exist_ok=True)
 _CACHE_DIR = Path(tempfile.gettempdir()) / "zizmor-benchmark-cache"
 _CACHE_DIR.mkdir(exist_ok=True)
 
+_GH_TOKEN = os.getenv("GH_TOKEN")
+
 
 class Log:
     def __init__(self, scope: str | None) -> None:
@@ -110,6 +113,7 @@ class Benchmark(TypedDict):
     source: str
     source_sha256: str
     stencil: str
+    online: bool | None
 
 
 Plan = list[str]
@@ -129,6 +133,10 @@ class Bench:
             case _:
                 LOG.error(f"Unknown source type: {self.benchmark['source_type']}")
 
+        if self.benchmark.get("online", False):
+            if not _GH_TOKEN:
+                LOG.error("Benchmark requires online access but GH_TOKEN is not set")
+
         stencil = self.benchmark["stencil"]
         command = stencil.replace("$ZIZMOR", str(_ZIZMOR)).replace(
             "$INPUTS", " ".join(inputs)
@@ -180,6 +188,9 @@ def main() -> None:
     parser.add_argument(
         "--dry-run", action="store_true", help="Show plans without running them"
     )
+    parser.add_argument(
+        "--offline", action="store_true", help="Run only offline benchmarks"
+    )
 
     args = parser.parse_args()
 
@@ -209,6 +220,10 @@ def main() -> None:
     benchmarks: list[Benchmark] = json.loads(_BENCHMARKS.read_text(encoding="utf-8"))
     LOG.info(f"found {len(benchmarks)} benchmarks in {_BENCHMARKS.name}")
 
+    if args.offline:
+        benchmarks = [b for b in benchmarks if not b.get("online", False)]
+        LOG.info(f"filtered to {len(benchmarks)} offline benchmarks")
+
     benches = [Bench(benchmark) for benchmark in benchmarks]
     plans = []
     with LOG.scope("plan"):

bench/benchmarks.json

@@ -12,5 +12,13 @@
         "source": "https://github.com/python/cpython/archive/48f88310044c6ef877f3b0761cf7afece2f8fb3a.zip",
         "source_sha256": "a52a67f1dd9cfa67c7d1305d5b9639629abe247b2c32f01b77f790ddf8b49503",
         "stencil": "$ZIZMOR --offline --format=plain --no-exit-codes --no-config $INPUTS"
+    },
+    {
+        "name": "gha-hazmat-e4094ce-online",
+        "source_type": "archive-url",
+        "source": "https://github.com/woodruffw/gha-hazmat/archive/e4094ce.zip",
+        "source_sha256": "ef038ba72dca71ba5913313890feb65b650363bd5640cc33cb5994af6189d311",
+        "stencil": "$ZIZMOR --format=plain --no-exit-codes --no-config $INPUTS",
+        "online": true
     }
 ]

docs/development.md

@@ -179,15 +179,23 @@ Benchmarks are stored in the top-level `bench/` directory, and can be
 run locally with:
 
 ```bash
-# run all benchmarks
+# run all offline benchmarks
 make bench
 ```
 
-We currently run benchmarks in the CI and report their results
+We currently run offline benchmarks in the CI and report their results
 to [Bencher](https://bencher.dev/). See
 [our project page](https://bencher.dev/console/projects/zizmor/plots)
 on Bencher for results and trends.
 
+There are also online benchmarks, but these don't get run automatically.
+To run them, you can pass `GH_TOKEN` to the `bench/benchmark.py` script
+directly:
+
+```bash
+GH_TOKEN=$(gh auth token) uv run bench/benchmark.py
+```
+
 ### Adding new benchmarks
 
 `zizmor` currently orchestrates benchmarks with `bench/benchmark.py`,