mirror of
https://github.com/astral-sh/ruff.git
synced 2025-08-04 02:38:25 +00:00
[red-knot] Markdown test runner (#15632)
## Summary As more and more tests move to Markdown, running the mdtest suite becomes one of the most common tasks for developers working on Red Knot. There are a few pain points when doing so, however: - The `build.rs` script enforces recompilation (~five seconds) whenever something changes in the `resource/mdtest` folder. This is strictly necessary, because whenever files are added or removed, the test harness needs to be updated. But this is very rarely the case! The most common scenario is that a Markdown file has *changed*, and in this case, no recompilation is necessary. It is currently not possible to distinguish these two cases using `cargo::rerun-if-changed`. One can work around this by running the test executable manually, but it requires finding the path to the correct `mdtest-<random-hash>` executable. - All Markdown tests are run by default. This is needed whenever Rust code changes, but while working on the tests themselves, it is often much more convenient to only run the tests for a single file. This can be done by using a `mdtest__path_to_file` filter, but this needs to be manually spelled out or copied from the test output. - `cargo`s test output for a failing Markdown test is often unnecessarily verbose. Unless there is an *actual* panic somewhere in the code, mdtests usually fail with the explicit *"Some tests failed"* panic in the mdtest suite. But in those cases, we are not interested in the pointer to the source of this panic, but only in the mdtest suite output. This PR adds a Markdown test runner tool that attempts to make the developer experience better. Once it is started using ```bash uv run -q crates/red_knot_python_semantic/mdtest.py ``` it will first recompile the tests once (if cargo requires it), find the path to the `mdtest` executable, and then enter into a mode where it watches for changes in the `red_knot_python_semantic` crate. Whenever … * … a Markdown file changes, it will rerun the mdtest for this specific file automatically (no recompilation!). * … a Markdown file is added, it will recompile the tests and then run the mdtest for the new file * … Rust code is changed, it will recompile the tests and run all of them The tool also trims down `cargo test` output and only shows the actual mdtest errors. The tool will certainly require a few more iterations before it becomes mature, but I'm curious to hear if there is any interest for something like this. ## Test Plan - Tested the new runner under various scenarios. --------- Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
David Peter
GitHub
parent
187a358d7a
commit
4656e3c90f
3 changed files with 365 additions and 0 deletions
209
crates/red_knot_python_semantic/mdtest.py
Normal file
209
crates/red_knot_python_semantic/mdtest.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
"""A runner for Markdown-based tests for Red Knot"""
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "rich",
|
||||
# "watchfiles",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal, Never, assert_never
|
||||
|
||||
from rich.console import Console
|
||||
from watchfiles import Change, watch
|
||||
|
||||
CRATE_NAME: Final = "red_knot_python_semantic"
|
||||
CRATE_ROOT: Final = Path(__file__).resolve().parent
|
||||
MDTEST_DIR: Final = CRATE_ROOT / "resources" / "mdtest"
|
||||
|
||||
|
||||
class MDTestRunner:
|
||||
mdtest_executable: Path | None
|
||||
console: Console
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.mdtest_executable = None
|
||||
self.console = Console()
|
||||
|
||||
def _run_cargo_test(self, *, message_format: Literal["human", "json"]) -> str:
|
||||
return subprocess.check_output(
|
||||
[
|
||||
"cargo",
|
||||
"test",
|
||||
"--package",
|
||||
CRATE_NAME,
|
||||
"--no-run",
|
||||
"--color=always",
|
||||
"--message-format",
|
||||
message_format,
|
||||
],
|
||||
cwd=CRATE_ROOT,
|
||||
env=dict(os.environ, CLI_COLOR="1"),
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def _recompile_tests(
|
||||
self, status_message: str, *, message_on_success: bool = True
|
||||
) -> bool:
|
||||
with self.console.status(status_message):
|
||||
# Run it with 'human' format in case there are errors:
|
||||
try:
|
||||
self._run_cargo_test(message_format="human")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(e.output)
|
||||
return False
|
||||
|
||||
# Run it again with 'json' format to find the mdtest executable:
|
||||
json_output = self._run_cargo_test(message_format="json")
|
||||
|
||||
if json_output:
|
||||
self._get_executable_path_from_json(json_output)
|
||||
|
||||
if message_on_success:
|
||||
self.console.print("[dim]Tests compiled successfully[/dim]")
|
||||
return True
|
||||
|
||||
def _get_executable_path_from_json(self, json_output: str) -> None:
|
||||
for json_line in json_output.splitlines():
|
||||
try:
|
||||
data = json.loads(json_line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if data.get("target", {}).get("name") == "mdtest":
|
||||
self.mdtest_executable = Path(data["executable"])
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Could not find mdtest executable after successful compilation"
|
||||
)
|
||||
|
||||
def _run_mdtest(
|
||||
self, arguments: list[str] | None = None, *, capture_output: bool = False
|
||||
) -> subprocess.CompletedProcess:
|
||||
assert self.mdtest_executable is not None
|
||||
|
||||
arguments = arguments or []
|
||||
return subprocess.run(
|
||||
[self.mdtest_executable, *arguments],
|
||||
cwd=CRATE_ROOT,
|
||||
env=dict(os.environ, CLICOLOR_FORCE="1"),
|
||||
capture_output=capture_output,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _run_mdtests_for_file(self, markdown_file: Path) -> None:
|
||||
path_mangled = (
|
||||
markdown_file.as_posix()
|
||||
.replace("/", "_")
|
||||
.replace("-", "_")
|
||||
.removesuffix(".md")
|
||||
)
|
||||
test_name = f"mdtest__{path_mangled}"
|
||||
|
||||
output = self._run_mdtest(["--exact", test_name], capture_output=True)
|
||||
|
||||
if output.returncode == 0:
|
||||
if "running 0 tests\n" in output.stdout:
|
||||
self.console.log(
|
||||
f"[yellow]Warning[/yellow]: No tests were executed with filter '{test_name}'"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f"Test for [bold green]{markdown_file}[/bold green] succeeded"
|
||||
)
|
||||
else:
|
||||
self.console.print()
|
||||
self.console.rule(
|
||||
f"Test for [bold red]{markdown_file}[/bold red] failed",
|
||||
style="gray",
|
||||
)
|
||||
# Skip 'cargo test' boilerplate at the beginning:
|
||||
lines = output.stdout.splitlines()
|
||||
start_index = 0
|
||||
for i, line in enumerate(lines):
|
||||
if f"{test_name} stdout" in line:
|
||||
start_index = i
|
||||
break
|
||||
|
||||
for line in lines[start_index + 1 :]:
|
||||
if "MDTEST_TEST_FILTER" in line:
|
||||
continue
|
||||
if line.strip() == "-" * 50:
|
||||
# Skip 'cargo test' boilerplate at the end
|
||||
break
|
||||
|
||||
print(line)
|
||||
|
||||
def watch(self) -> Never:
|
||||
self._recompile_tests("Compiling tests...", message_on_success=False)
|
||||
self.console.print("[dim]Ready to watch for changes...[/dim]")
|
||||
|
||||
for changes in watch(CRATE_ROOT):
|
||||
new_md_files = set()
|
||||
changed_md_files = set()
|
||||
rust_code_has_changed = False
|
||||
|
||||
for change, path_str in changes:
|
||||
path = Path(path_str)
|
||||
|
||||
if path.suffix == ".rs":
|
||||
rust_code_has_changed = True
|
||||
continue
|
||||
|
||||
if path.suffix != ".md":
|
||||
continue
|
||||
|
||||
try:
|
||||
relative_path = Path(path).relative_to(MDTEST_DIR)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
match change:
|
||||
case Change.added:
|
||||
# When saving a file, some editors (looking at you, Vim) might first
|
||||
# save the file with a temporary name (e.g. `file.md~`) and then rename
|
||||
# it to the final name. This creates a `deleted` and `added` change.
|
||||
# We treat those files as `changed` here.
|
||||
if (Change.deleted, path_str) in changes:
|
||||
changed_md_files.add(relative_path)
|
||||
else:
|
||||
new_md_files.add(relative_path)
|
||||
case Change.modified:
|
||||
changed_md_files.add(relative_path)
|
||||
case Change.deleted:
|
||||
# No need to do anything when a Markdown test is deleted
|
||||
pass
|
||||
case _ as unreachable:
|
||||
assert_never(unreachable)
|
||||
|
||||
if rust_code_has_changed:
|
||||
if self._recompile_tests("Rust code has changed, recompiling tests..."):
|
||||
self._run_mdtest()
|
||||
elif new_md_files:
|
||||
files = " ".join(file.as_posix() for file in new_md_files)
|
||||
self._recompile_tests(
|
||||
f"New Markdown test [yellow]{files}[/yellow] detected, recompiling tests..."
|
||||
)
|
||||
|
||||
for path in new_md_files | changed_md_files:
|
||||
self._run_mdtests_for_file(path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
runner = MDTestRunner()
|
||||
runner.watch()
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue