[ty] Add filtering option for mdtest runner (#21422)

## Summary

This change to the mdtest runner makes it easy to run on a subset of
tests/files. For example:

```
▶ uv run crates/ty_python_semantic/mdtest.py implicit
running 1 test
test mdtest__implicit_type_aliases ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 281 filtered out; finished in 0.83s

Ready to watch for changes...
```

Subsequent changes to either that test file or the Rust source code will
also only rerun the `implicit_type_aliases` test.

Multiple arguments can be provided, and filters can either be partial
file paths (`loops/for.md`, `loops/for`, `for`) or mangled test names
(`loops_for`):
```
▶ uv run crates/ty_python_semantic/mdtest.py implicit binary/union

running 2 tests
test mdtest__binary_unions ... ok
test mdtest__implicit_type_aliases ... ok

test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 280 filtered out; finished in 0.85s

Ready to watch for changes...
```

## Test Plan

Tested it interactively for a while
This commit is contained in:
David Peter 2025-11-13 13:20:31 +01:00 committed by GitHub
parent cd183c5e1f
commit d64b2f747c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -9,11 +9,12 @@
from __future__ import annotations from __future__ import annotations
import argparse
import json import json
import os import os
import subprocess import subprocess
from pathlib import Path from pathlib import Path
from typing import Final, Literal, Never, assert_never from typing import Final, Literal, assert_never
from rich.console import Console from rich.console import Console
from watchfiles import Change, watch from watchfiles import Change, watch
@ -32,10 +33,15 @@ MDTEST_DIR: Final = CRATE_ROOT / "resources" / "mdtest"
class MDTestRunner: class MDTestRunner:
mdtest_executable: Path | None mdtest_executable: Path | None
console: Console console: Console
filters: list[str]
def __init__(self) -> None: def __init__(self, filters: list[str] | None = None) -> None:
self.mdtest_executable = None self.mdtest_executable = None
self.console = Console() self.console = Console()
self.filters = [
f.removesuffix(".md").replace("/", "_").replace("-", "_")
for f in (filters or [])
]
def _run_cargo_test(self, *, message_format: Literal["human", "json"]) -> str: def _run_cargo_test(self, *, message_format: Literal["human", "json"]) -> str:
return subprocess.check_output( return subprocess.check_output(
@ -117,13 +123,16 @@ class MDTestRunner:
check=False, check=False,
) )
def _run_mdtests_for_file(self, markdown_file: Path) -> None: def _mangle_path(self, markdown_file: Path) -> str:
path_mangled = ( return (
markdown_file.as_posix() markdown_file.as_posix()
.replace("/", "_") .replace("/", "_")
.replace("-", "_") .replace("-", "_")
.removesuffix(".md") .removesuffix(".md")
) )
def _run_mdtests_for_file(self, markdown_file: Path) -> None:
path_mangled = self._mangle_path(markdown_file)
test_name = f"mdtest__{path_mangled}" test_name = f"mdtest__{path_mangled}"
output = self._run_mdtest(["--exact", test_name], capture_output=True) output = self._run_mdtest(["--exact", test_name], capture_output=True)
@ -165,9 +174,9 @@ class MDTestRunner:
print(line) print(line)
def watch(self) -> Never: def watch(self):
self._recompile_tests("Compiling tests...", message_on_success=False) self._recompile_tests("Compiling tests...", message_on_success=False)
self._run_mdtest() self._run_mdtest(self.filters)
self.console.print("[dim]Ready to watch for changes...[/dim]") self.console.print("[dim]Ready to watch for changes...[/dim]")
for changes in watch(*DIRS_TO_WATCH): for changes in watch(*DIRS_TO_WATCH):
@ -214,12 +223,12 @@ class MDTestRunner:
if rust_code_has_changed: if rust_code_has_changed:
if self._recompile_tests("Rust code has changed, recompiling tests..."): if self._recompile_tests("Rust code has changed, recompiling tests..."):
self._run_mdtest() self._run_mdtest(self.filters)
elif vendored_typeshed_has_changed: elif vendored_typeshed_has_changed:
if self._recompile_tests( if self._recompile_tests(
"Vendored typeshed has changed, recompiling tests..." "Vendored typeshed has changed, recompiling tests..."
): ):
self._run_mdtest() self._run_mdtest(self.filters)
elif new_md_files: elif new_md_files:
files = " ".join(file.as_posix() for file in new_md_files) files = " ".join(file.as_posix() for file in new_md_files)
self._recompile_tests( self._recompile_tests(
@ -231,8 +240,19 @@ class MDTestRunner:
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(
description="A runner for Markdown-based tests for ty"
)
parser.add_argument(
"filters",
nargs="*",
help="Partial paths or mangled names, e.g., 'loops/for.md' or 'loops_for'",
)
args = parser.parse_args()
try: try:
runner = MDTestRunner() runner = MDTestRunner(filters=args.filters)
runner.watch() runner.watch()
except KeyboardInterrupt: except KeyboardInterrupt:
print() print()