language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-29 05:15:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Rename Red Knot (#17820)

Browse source
This commit is contained in:
Micha Reiser 2025-05-03 19:49:15 +02:00 • committed by GitHub
parent e6a798b962
commit b51c4f82ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1564 changed files with 1598 additions and 1578 deletions
Download patch file Download diff file Expand all files Collapse all files

76
scripts/ty_benchmark/src/benchmark/__init__.py Normal file
View file

@ -0,0 +1,76 @@
from __future__ import annotations
import logging
import shlex
import subprocess
import typing
from pathlib import Path
class Command(typing.NamedTuple):
name: str
"""The name of the command to benchmark."""
command: list[str]
"""The command to benchmark."""
prepare: str | None = None
"""The command to run before each benchmark run."""
class Hyperfine(typing.NamedTuple):
name: str
"""The benchmark to run."""
commands: list[Command]
"""The commands to benchmark."""
warmup: int
"""The number of warmup runs to perform."""
min_runs: int
"""The minimum number of runs to perform."""
verbose: bool
"""Whether to print verbose output."""
json: bool
"""Whether to export results to JSON."""
def run(self, *, cwd: Path | None = None) -> None:
"""Run the benchmark using `hyperfine`."""
args = [
"hyperfine",
# Most repositories have some typing errors.
# This is annoying because it prevents us from capturing "real" errors.
"-i",
]
# Export to JSON.
if self.json:
args.extend(["--export-json", f"{self.name}.json"])
# Preamble: benchmark-wide setup.
if self.verbose:
args.append("--show-output")
args.extend(["--warmup", str(self.warmup), "--min-runs", str(self.min_runs)])
# Add all command names,
for command in self.commands:
args.extend(["--command-name", command.name])
# Add all prepare statements.
for command in self.commands:
args.extend(["--prepare", command.prepare or ""])
# Add all commands.
for command in self.commands:
args.append(shlex.join(command.command))
logging.info(f"Running {args}")
subprocess.run(
args,
cwd=cwd,
)

217
scripts/ty_benchmark/src/benchmark/cases.py Normal file
View file

@ -0,0 +1,217 @@
from __future__ import annotations
import abc
import enum
import logging
import os
import shutil
import subprocess
import sys
from pathlib import Path
from benchmark import Command
from benchmark.projects import Project
class Benchmark(enum.Enum):
"""Enumeration of the benchmarks to run."""
COLD = "cold"
"""Cold check of an entire project without a cache present."""
WARM = "warm"
"""Re-checking the entire project without any changes"."""
def which_tool(name: str) -> Path:
tool = shutil.which(name)
assert tool is not None, (
f"Tool {name} not found. Run the script with `uv run <script>`."
)
return Path(tool)
class Tool(abc.ABC):
def command(
self, benchmark: Benchmark, project: Project, venv: Venv
) -> Command | None:
"""Generate a command to benchmark a given tool."""
match benchmark:
case Benchmark.COLD:
return self.cold_command(project, venv)
case Benchmark.WARM:
return self.warm_command(project, venv)
case _:
raise ValueError(f"Invalid benchmark: {benchmark}")
@abc.abstractmethod
def cold_command(self, project: Project, venv: Venv) -> Command: ...
def warm_command(self, project: Project, venv: Venv) -> Command | None:
return None
class Ty(Tool):
path: Path
name: str
def __init__(self, *, path: Path | None = None):
self.name = str(path) or "ty"
self.path = path or (
(Path(__file__) / "../../../../../target/release/ty").resolve()
)
assert self.path.is_file(), (
f"ty not found at '{self.path}'. Run `cargo build --release --bin ty`."
)
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [str(self.path), "check", "-v", *project.include]
command.extend(["--python", str(venv.path)])
return Command(
name="ty",
command=command,
)
class Mypy(Tool):
path: Path
def __init__(self, *, path: Path | None = None):
self.path = path or which_tool(
"mypy",
)
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [
*self._base_command(project, venv),
"--no-incremental",
"--cache-dir",
os.devnull,
]
return Command(
name="mypy",
command=command,
)
def warm_command(self, project: Project, venv: Venv) -> Command | None:
command = [
str(self.path),
*(project.mypy_arguments or project.include),
"--python-executable",
str(venv.python),
]
return Command(
name="mypy",
command=command,
)
def _base_command(self, project: Project, venv: Venv) -> list[str]:
return [
str(self.path),
"--python-executable",
str(venv.python),
*(project.mypy_arguments or project.include),
]
class Pyright(Tool):
path: Path
def __init__(self, *, path: Path | None = None):
self.path = path or which_tool("pyright")
def cold_command(self, project: Project, venv: Venv) -> Command:
command = [
str(self.path),
"--threads",
"--venvpath",
str(
venv.path.parent
), # This is not the path to the venv folder, but the folder that contains the venv...
*(project.pyright_arguments or project.include),
]
return Command(
name="Pyright",
command=command,
)
class Venv:
path: Path
def __init__(self, path: Path):
self.path = path
@property
def name(self) -> str:
"""The name of the virtual environment directory."""
return self.path.name
@property
def python(self) -> Path:
"""Returns the path to the python executable"""
return self.script("python")
@property
def bin(self) -> Path:
bin_dir = "scripts" if sys.platform == "win32" else "bin"
return self.path / bin_dir
def script(self, name: str) -> Path:
extension = ".exe" if sys.platform == "win32" else ""
return self.bin / f"{name}{extension}"
@staticmethod
def create(parent: Path) -> Venv:
"""Creates a new, empty virtual environment."""
command = [
"uv",
"venv",
"--quiet",
"venv",
]
try:
subprocess.run(
command, cwd=parent, check=True, capture_output=True, text=True
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to create venv: {e.stderr}")
root = parent / "venv"
return Venv(root)
def install(self, dependencies: list[str]) -> None:
"""Installs the dependencies required to type check the project."""
logging.debug(f"Installing dependencies: {', '.join(dependencies)}")
command = [
"uv",
"pip",
"install",
"--python",
self.python.as_posix(),
"--quiet",
# We pass `--exclude-newer` to ensure that type-checking of one of
# our projects isn't unexpectedly broken by a change in the
# annotations of one of that project's dependencies
"--exclude-newer",
"2024-09-03T00:00:00Z",
*dependencies,
]
try:
subprocess.run(
command, cwd=self.path, check=True, capture_output=True, text=True
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to install dependencies: {e.stderr}")

145
scripts/ty_benchmark/src/benchmark/projects.py Normal file
View file

@ -0,0 +1,145 @@
import logging
import subprocess
import typing
from pathlib import Path
class Project(typing.NamedTuple):
name: str
"""The name of the project to benchmark."""
repository: str
"""The git repository to clone."""
revision: str
dependencies: list[str]
"""List of type checking dependencies.
Dependencies are pinned using a `--exclude-newer` flag when installing them
into the virtual environment; see the `Venv.install()` method for details.
"""
include: list[str] = []
"""The directories and files to check. If empty, checks the current directory"""
pyright_arguments: list[str] | None = None
"""The arguments passed to pyright. Overrides `include` if set."""
mypy_arguments: list[str] | None = None
"""The arguments passed to mypy. Overrides `include` if set."""
def clone(self, checkout_dir: Path) -> None:
# Skip cloning if the project has already been cloned (the script doesn't yet support updating)
if (checkout_dir / ".git").exists():
return
logging.debug(f"Cloning {self.repository} to {checkout_dir}")
try:
# git doesn't support cloning a specific revision.
# This is the closest that I found to a "shallow clone with a specific revision"
subprocess.run(
[
"git",
"init",
"--quiet",
],
env={"GIT_TERMINAL_PROMPT": "0"},
cwd=checkout_dir,
check=True,
capture_output=True,
text=True,
)
subprocess.run(
["git", "remote", "add", "origin", str(self.repository), "--no-fetch"],
env={"GIT_TERMINAL_PROMPT": "0"},
cwd=checkout_dir,
check=True,
capture_output=True,
text=True,
)
subprocess.run(
[
"git",
"fetch",
"origin",
self.revision,
"--quiet",
"--depth",
"1",
"--no-tags",
],
check=True,
cwd=checkout_dir,
capture_output=True,
text=True,
)
subprocess.run(
["git", "reset", "--hard", "FETCH_HEAD", "--quiet"],
check=True,
cwd=checkout_dir,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to clone {self.name}: {e.stderr}")
logging.info(f"Cloned {self.name} to {checkout_dir}.")
# Selection of projects taken from
# [mypy-primer](https://github.com/hauntsaninja/mypy_primer/blob/0ea6cc614b3e91084059b9a3acc58f94c066a211/mypy_primer/projects.py#L71).
# May require frequent updating, especially the dependencies list
ALL = [
Project(
name="black",
repository="https://github.com/psf/black",
revision="ac28187bf4a4ac159651c73d3a50fe6d0f653eac",
include=["src"],
dependencies=[
"aiohttp",
"click",
"pathspec",
"tomli",
"platformdirs",
"packaging",
],
),
Project(
name="jinja",
repository="https://github.com/pallets/jinja",
revision="b490da6b23b7ad25dc969976f64dc4ffb0a2c182",
include=[],
dependencies=["markupsafe"],
),
Project(
name="pandas",
repository="https://github.com/pandas-dev/pandas",
revision="7945e563d36bcf4694ccc44698829a6221905839",
include=["pandas"],
dependencies=[
"numpy",
"types-python-dateutil",
"types-pytz",
"types-PyMySQL",
"types-setuptools",
"pytest",
],
),
Project(
name="isort",
repository="https://github.com/pycqa/isort",
revision="7de182933fd50e04a7c47cc8be75a6547754b19c",
mypy_arguments=["--ignore-missing-imports", "isort"],
include=["isort"],
dependencies=["types-setuptools"],
),
]
DEFAULT: list[str] = ["black", "jinja", "isort"]

153
scripts/ty_benchmark/src/benchmark/run.py Normal file
View file

@ -0,0 +1,153 @@
from __future__ import annotations
import argparse
import json
import logging
import tempfile
import typing
from pathlib import Path
from benchmark import Hyperfine
from benchmark.cases import Benchmark, Mypy, Pyright, Tool, Ty, Venv
from benchmark.projects import ALL as all_projects
from benchmark.projects import DEFAULT as default_projects
if typing.TYPE_CHECKING:
from benchmark.cases import Tool
def main() -> None:
"""Run the benchmark."""
parser = argparse.ArgumentParser(
description="Benchmark ty against other packaging tools."
)
parser.add_argument(
"--verbose", "-v", action="store_true", help="Print verbose output."
)
parser.add_argument(
"--warmup",
type=int,
help="The number of warmup runs to perform.",
default=3,
)
parser.add_argument(
"--min-runs",
type=int,
help="The minimum number of runs to perform.",
default=10,
)
parser.add_argument(
"--benchmark",
"-b",
type=str,
help="The benchmark(s) to run.",
choices=[benchmark.value for benchmark in Benchmark],
action="append",
)
parser.add_argument(
"--project",
"-p",
type=str,
help="The project(s) to run.",
choices=[project.name for project in all_projects],
action="append",
)
parser.add_argument(
"--mypy",
help="Whether to benchmark `mypy`.",
action="store_true",
)
parser.add_argument(
"--pyright",
help="Whether to benchmark `pyright`.",
action="store_true",
)
parser.add_argument(
"--ty",
help="Whether to benchmark ty (assumes a ty binary exists at `./target/release/ty`).",
action="store_true",
)
parser.add_argument(
"--ty-path",
type=Path,
help="Path(s) to the ty binary to benchmark.",
action="append",
)
args = parser.parse_args()
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARN,
format="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
verbose = args.verbose
warmup = args.warmup
min_runs = args.min_runs
# Determine the tools to benchmark, based on the user-provided arguments.
suites: list[Tool] = []
if args.pyright:
suites.append(Pyright())
if args.ty:
suites.append(Ty())
for path in args.ty_path or []:
suites.append(Ty(path=path))
if args.mypy:
suites.append(Mypy())
# If no tools were specified, default to benchmarking all tools.
suites = suites or [Ty(), Pyright(), Mypy()]
# Determine the benchmarks to run, based on user input.
benchmarks = (
[Benchmark(benchmark) for benchmark in args.benchmark]
if args.benchmark is not None
else list(Benchmark)
)
projects = [
project
for project in all_projects
if project.name in (args.project or default_projects)
]
for project in projects:
with tempfile.TemporaryDirectory() as tempdir:
cwd = Path(tempdir)
project.clone(cwd)
venv = Venv.create(cwd)
venv.install(project.dependencies)
# Set the `venv` config for pyright. Pyright only respects the `--venvpath`
# CLI option when `venv` is set in the configuration... 🤷‍♂️
with open(cwd / "pyrightconfig.json", "w") as f:
f.write(json.dumps(dict(venv=venv.name)))
for benchmark in benchmarks:
# Generate the benchmark command for each tool.
commands = [
command
for suite in suites
if (command := suite.command(benchmark, project, venv))
]
# not all tools support all benchmarks.
if not commands:
continue
print(f"{project.name} ({benchmark.value})")
hyperfine = Hyperfine(
name=f"{project.name}-{benchmark.value}",
commands=commands,
warmup=warmup,
min_runs=min_runs,
verbose=verbose,
json=False,
)
hyperfine.run(cwd=cwd)