mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 10:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add basic red knot benchmark (#13026)

Browse source 
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
Micha Reiser 2024-08-23 08:22:42 +02:00 • committed by GitHub
parent 1ca14e4335
commit 4f6accb5c6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 699 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

21
scripts/knot_benchmark/README.md Normal file
View file

@ -0,0 +1,21 @@
## Getting started
1. [Install `uv`](https://docs.astral.sh/uv/getting-started/installation/)
- Unix: `curl -LsSf https://astral.sh/uv/install.sh | sh`
- Windows: `powershell -c "irm https://astral.sh/uv/install.ps1 | iex"`
1. Build red_knot: `cargo build --bin red_knot --release`
1. `cd` into the benchmark directory: `cd scripts/knot_benchmark`
1. Run benchmarks: `uv run benchmark`
## Known limitations
Red Knot only implements a tiny fraction of Mypy's and Pyright's functionality,
so the benchmarks aren't in any way a fair comparison today. However,
they'll become more meaningful as we build out more type checking features in Red Knot.
### Windows support
The script should work on Windows, but we haven't tested it yet.
We do make use of `shlex` which has known limitations when using non-POSIX shells.

21
scripts/knot_benchmark/pyproject.toml Normal file
View file

@ -0,0 +1,21 @@
[project]
name = "knot_benchmark"
version = "0.0.1"
description = "Package for running end-to-end Red Knot benchmarks"
requires-python = ">=3.12"
dependencies = ["mypy", "pyright"]
[project.scripts]
benchmark = "benchmark.run:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/benchmark"]
[tool.ruff.lint]
ignore = [
"E501", # We use ruff format
]

76
scripts/knot_benchmark/src/benchmark/__init__.py Normal file
View file

@ -0,0 +1,76 @@
from __future__ import annotations
import logging
import shlex
import subprocess
import typing
from pathlib import Path
class Command(typing.NamedTuple):
name: str
"""The name of the command to benchmark."""
command: list[str]
"""The command to benchmark."""
prepare: str | None = None
"""The command to run before each benchmark run."""
class Hyperfine(typing.NamedTuple):
name: str
"""The benchmark to run."""
commands: list[Command]
"""The commands to benchmark."""
warmup: int
"""The number of warmup runs to perform."""
min_runs: int
"""The minimum number of runs to perform."""
verbose: bool
"""Whether to print verbose output."""
json: bool
"""Whether to export results to JSON."""
def run(self, *, cwd: Path | None = None) -> None:
"""Run the benchmark using `hyperfine`."""
args = [
"hyperfine",
# Most repositories have some typing errors.
# This is annoying because it prevents us from capturing "real" errors.
"-i",
]
# Export to JSON.
if self.json:
args.extend(["--export-json", f"{self.name}.json"])
# Preamble: benchmark-wide setup.
if self.verbose:
args.append("--show-output")
args.extend(["--warmup", str(self.warmup), "--min-runs", str(self.min_runs)])
# Add all command names,
for command in self.commands:
args.extend(["--command-name", command.name])
# Add all prepare statements.
for command in self.commands:
args.extend(["--prepare", command.prepare or ""])
# Add all commands.
for command in self.commands:
args.append(shlex.join(command.command))
logging.info(f"Running {args}")
subprocess.run(
args,
cwd=cwd,
)

212
scripts/knot_benchmark/src/benchmark/cases.py Normal file
View file

@ -0,0 +1,212 @@
from __future__ import annotations
import abc
import enum
import logging
import os
import shutil
import subprocess
import sys
from pathlib import Path
from benchmark import Command
from benchmark.projects import Project
class Benchmark(enum.Enum):
"""Enumeration of the benchmarks to run."""
COLD = "cold"
"""Cold check of an entire project without a cache present."""
WARM = "warm"
"""Re-checking the entire project without any changes"."""
def which_tool(name: str) -> Path:
tool = shutil.which(name)
assert (
tool is not None
), f"Tool {name} not found. Run the script with `uv run <script>`."
return Path(tool)
class Tool(abc.ABC):
def command(
self, benchmark: Benchmark, project: Project, venv: Venv
) -> Command | None:
"""Generate a command to benchmark a given tool."""
match benchmark:
case Benchmark.COLD:
return self.cold_command(project, venv)
case Benchmark.WARM:
return self.warm_command(project, venv)
case _:
raise ValueError(f"Invalid benchmark: {benchmark}")
@abc.abstractmethod
def cold_command(self, project: Project, venv: Venv) -> Command: ...
def warm_command(self, project: Project, venv: Venv) -> Command | None:
return None
class Knot(Tool):
path: Path
name: str
def __init__(self, *, path: Path | None = None):
self.name = str(path) or "knot"
self.path = path or (
(Path(__file__) / "../../../../../target/release/red_knot").resolve()
)
assert self.path.is_file(), f"Red Knot not found at '{self.path}'. Run `cargo build --release --bin red_knot`."
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [str(self.path), "-v"]
assert len(project.include) < 2, "Knot doesn't support multiple source folders"
if project.include:
command.extend(["--current-directory", project.include[0]])
command.extend(["--venv-path", str(venv.path)])
return Command(
name="knot",
command=command,
)
class Mypy(Tool):
path: Path
def __init__(self, *, path: Path | None = None):
self.path = path or which_tool(
"mypy",
)
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [
*self._base_command(project, venv),
"--no-incremental",
"--cache-dir",
os.devnull,
]
return Command(
name="mypy",
command=command,
)
def warm_command(self, project: Project, venv: Venv) -> Command | None:
command = [
str(self.path),
*(project.mypy_arguments or project.include),
"--python-executable",
str(venv.python),
]
return Command(
name="mypy",
command=command,
)
def _base_command(self, project: Project, venv: Venv) -> list[str]:
return [
str(self.path),
"--python-executable",
str(venv.python),
*(project.mypy_arguments or project.include),
]
class Pyright(Tool):
path: Path
def __init__(self, *, path: Path | None = None):
self.path = path or which_tool("pyright")
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [
str(self.path),
"--venvpath",
str(
venv.path.parent
), # This is not the path to the venv folder, but the folder that contains the venv...
*(project.pyright_arguments or project.include),
]
return Command(
name="Pyright",
command=command,
)
class Venv:
path: Path
def __init__(self, path: Path):
self.path = path
@property
def name(self):
"""The name of the virtual environment directory."""
return self.path.name
@property
def python(self):
"""Returns the path to the python executable"""
return self.script("python")
@property
def bin(self) -> Path:
bin_dir = "scripts" if sys.platform == "win32" else "bin"
return self.path / bin_dir
def script(self, name: str) -> Path:
extension = ".exe" if sys.platform == "win32" else ""
return self.bin / f"{name}{extension}"
@staticmethod
def create(parent: Path) -> Venv:
"""Creates a new, empty virtual environment."""
command = [
"uv",
"venv",
"--quiet",
"venv",
]
try:
subprocess.run(
command, cwd=parent, check=True, capture_output=True, text=True
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to create venv: {e.stderr}")
root = parent / "venv"
return Venv(root)
def install(self, dependencies: list[str]):
"""Installs the dependencies required to type check the project."""
logging.debug(f"Installing dependencies: {', '.join(dependencies)}")
command = [
"uv",
"pip",
"install",
"--quiet",
*dependencies,
]
try:
subprocess.run(
command, cwd=self.path, check=True, capture_output=True, text=True
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to install dependencies: {e.stderr}")

142
scripts/knot_benchmark/src/benchmark/projects.py Normal file
View file

@ -0,0 +1,142 @@
import logging
import os
import subprocess
import typing
from pathlib import Path
class Project(typing.NamedTuple):
name: str
"""The name of the project to benchmark."""
repository: str
"""The git repository to clone."""
revision: str
dependencies: list[str]
"""List of type checking dependencies"""
include: list[str] = []
"""The directories and files to check. If empty, checks the current directory"""
pyright_arguments: list[str] | None = None
"""The arguments passed to pyright. Overrides `include` if set."""
mypy_arguments: list[str] | None = None
"""The arguments passed to mypy. Overrides `include` if set."""
def clone(self, checkout_dir: Path):
# Skip cloning if the project has already been cloned (the script doesn't yet support updating)
if os.path.exists(os.path.join(checkout_dir, ".git")):
return
logging.debug(f"Cloning {self.repository} to {checkout_dir}")
try:
# git doesn't support cloning a specific revision.
# This is the closest that I found to a "shallow clone with a specific revision"
subprocess.run(
[
"git",
"init",
"--quiet",
],
env={"GIT_TERMINAL_PROMPT": "0"},
cwd=checkout_dir,
check=True,
capture_output=True,
text=True,
)
subprocess.run(
["git", "remote", "add", "origin", str(self.repository), "--no-fetch"],
env={"GIT_TERMINAL_PROMPT": "0"},
cwd=checkout_dir,
check=True,
capture_output=True,
text=True,
)
subprocess.run(
[
"git",
"fetch",
"origin",
self.revision,
"--quiet",
"--depth",
"1",
"--no-tags",
],
check=True,
cwd=checkout_dir,
capture_output=True,
text=True,
)
subprocess.run(
["git", "reset", "--hard", "FETCH_HEAD", "--quiet"],
check=True,
cwd=checkout_dir,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to clone {self.name}: {e.stderr}")
logging.info(f"Cloned {self.name} to {checkout_dir}.")
# Selection of projects taken from
# [mypy-primer](https://github.com/hauntsaninja/mypy_primer/blob/0ea6cc614b3e91084059b9a3acc58f94c066a211/mypy_primer/projects.py#L71).
# May require frequent updating, especially the dependencies list
ALL = [
Project(
name="black",
repository="https://github.com/psf/black",
revision="c20423249e9d8dfb8581eebbfc67a13984ee45e9",
include=["src"],
dependencies=[
"aiohttp",
"click",
"pathspec",
"tomli",
"platformdirs",
"packaging",
],
),
Project(
name="jinja",
repository="https://github.com/pallets/jinja",
revision="b490da6b23b7ad25dc969976f64dc4ffb0a2c182",
include=[],
dependencies=["markupsafe"],
),
Project(
name="pandas",
repository="https://github.com/pandas-dev/pandas",
revision="7945e563d36bcf4694ccc44698829a6221905839",
include=["pandas"],
dependencies=[
"numpy",
"types-python-dateutil",
"types-pytz",
"types-PyMySQL",
"types-setuptools",
"pytest",
],
),
Project(
name="isort",
repository="https://github.com/pycqa/isort",
revision="7de182933fd50e04a7c47cc8be75a6547754b19c",
mypy_arguments=["--ignore-missing-imports", "isort"],
include=["isort"],
dependencies=["types-setuptools"],
),
]
DEFAULT: list[str] = ["black", "jinja", "isort"]

153
scripts/knot_benchmark/src/benchmark/run.py Normal file
View file

@ -0,0 +1,153 @@
from __future__ import annotations
import argparse
import json
import logging
import tempfile
import typing
from pathlib import Path
from benchmark import Hyperfine
from benchmark.cases import Benchmark, Knot, Mypy, Pyright, Tool, Venv
from benchmark.projects import ALL as all_projects
from benchmark.projects import DEFAULT as default_projects
if typing.TYPE_CHECKING:
from benchmark.cases import Tool
def main():
"""Run the benchmark."""
parser = argparse.ArgumentParser(
description="Benchmark knot against other packaging tools."
)
parser.add_argument(
"--verbose", "-v", action="store_true", help="Print verbose output."
)
parser.add_argument(
"--warmup",
type=int,
help="The number of warmup runs to perform.",
default=3,
)
parser.add_argument(
"--min-runs",
type=int,
help="The minimum number of runs to perform.",
default=10,
)
parser.add_argument(
"--benchmark",
"-b",
type=str,
help="The benchmark(s) to run.",
choices=[benchmark.value for benchmark in Benchmark],
action="append",
)
parser.add_argument(
"--project",
"-p",
type=str,
help="The project(s) to run.",
choices=[project.name for project in all_projects],
action="append",
)
parser.add_argument(
"--mypy",
help="Whether to benchmark `mypy`.",
action="store_true",
)
parser.add_argument(
"--pyright",
help="Whether to benchmark `pyright`.",
action="store_true",
)
parser.add_argument(
"--knot",
help="Whether to benchmark knot (assumes a red_knot binary exists at `./target/release/red_knot`).",
action="store_true",
)
parser.add_argument(
"--knot-path",
type=str,
help="Path(s) to the red_knot binary to benchmark.",
action="append",
)
args = parser.parse_args()
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARN,
format="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
verbose = args.verbose
warmup = args.warmup
min_runs = args.min_runs
# Determine the tools to benchmark, based on the user-provided arguments.
suites: list[Tool] = []
if args.pyright:
suites.append(Pyright())
if args.knot:
suites.append(Knot())
for path in args.knot_path or []:
suites.append(Knot(path=path))
if args.mypy:
suites.append(Mypy())
# If no tools were specified, default to benchmarking all tools.
suites = suites or [Knot(), Pyright(), Mypy()]
# Determine the benchmarks to run, based on user input.
benchmarks = (
[Benchmark(benchmark) for benchmark in args.benchmark]
if args.benchmark is not None
else list(Benchmark)
)
projects = [
project
for project in all_projects
if project.name in (args.project or default_projects)
]
for project in projects:
with tempfile.TemporaryDirectory() as cwd:
cwd = Path(cwd)
project.clone(cwd)
venv = Venv.create(cwd)
venv.install(project.dependencies)
# Set the `venv` config for pyright. Pyright only respects the `--venvpath`
# CLI option when `venv` is set in the configuration... 🤷‍♂️
with open(cwd / "pyrightconfig.json", "w") as f:
f.write(json.dumps(dict(venv=venv.name)))
for benchmark in benchmarks:
# Generate the benchmark command for each tool.
commands = [
command
for suite in suites
if (command := suite.command(benchmark, project, venv))
]
# not all tools support all benchmarks.
if not commands:
continue
print(f"{project.name} ({benchmark.value})")
hyperfine = Hyperfine(
name=f"{project.name}-{benchmark.value}",
commands=commands,
warmup=warmup,
min_runs=min_runs,
verbose=verbose,
json=False,
)
hyperfine.run(cwd=cwd)

74
scripts/knot_benchmark/uv.lock generated Normal file
View file

@ -0,0 +1,74 @@
version = 1
requires-python = ">=3.12"
[[package]]
name = "knot-benchmark"
version = "0.0.1"
source = { editable = "." }
dependencies = [
{ name = "mypy" },
{ name = "pyright" },
]
[package.metadata]
requires-dist = [
{ name = "mypy" },
{ name = "pyright" },
]
[[package]]
name = "mypy"
version = "1.11.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "mypy-extensions" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b6/9c/a4b3bda53823439cf395db8ecdda6229a83f9bf201714a68a15190bb2919/mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08", size = 3078369 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/34/69638cee2e87303f19a0c35e80d42757e14d9aba328f272fdcdc0bf3c9b8/mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8", size = 10995789 },
{ url = "https://files.pythonhosted.org/packages/c4/3c/3e0611348fc53a4a7c80485959478b4f6eae706baf3b7c03cafa22639216/mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a", size = 10002696 },
{ url = "https://files.pythonhosted.org/packages/1c/21/a6b46c91b4c9d1918ee59c305f46850cde7cbea748635a352e7c3c8ed204/mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417", size = 12505772 },
{ url = "https://files.pythonhosted.org/packages/c4/55/07904d4c8f408e70308015edcbff067eaa77514475938a9dd81b063de2a8/mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e", size = 12954190 },
{ url = "https://files.pythonhosted.org/packages/1e/b7/3a50f318979c8c541428c2f1ee973cda813bcc89614de982dafdd0df2b3e/mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525", size = 9663138 },
{ url = "https://files.pythonhosted.org/packages/f8/d4/4960d0df55f30a7625d9c3c9414dfd42f779caabae137ef73ffaed0c97b9/mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54", size = 2619257 },
]
[[package]]
name = "mypy-extensions"
version = "1.0.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 },
]
[[package]]
name = "nodeenv"
version = "1.9.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 },
]
[[package]]
name = "pyright"
version = "1.1.377"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nodeenv" },
]
sdist = { url = "https://files.pythonhosted.org/packages/49/f0/25b0db363d6888164adb7c828b877bbf2c30936955fb9513922ae03e70e4/pyright-1.1.377.tar.gz", hash = "sha256:aabc30fedce0ded34baa0c49b24f10e68f4bfc8f68ae7f3d175c4b0f256b4fcf", size = 17484 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/34/c9/89c40c4de44fe9463e77dddd0c4e2d2dd7a93e8ddc6858dfe7d5f75d263d/pyright-1.1.377-py3-none-any.whl", hash = "sha256:af0dd2b6b636c383a6569a083f8c5a8748ae4dcde5df7914b3f3f267e14dd162", size = 18223 },
]
[[package]]
name = "typing-extensions"
version = "4.12.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 },
]