Check docs formatting check (#4270)

.github/workflows/ci.yaml

@@ -237,5 +237,7 @@ jobs:
         run: python scripts/transform_readme.py --target mkdocs
       - name: "Generate docs"
         run: python scripts/generate_mkdocs.py
+      - name: "Check docs formatting"
+        run: python scripts/check_docs_formatted.py
       - name: "Build docs"
         run: mkdocs build --strict

crates/ruff/src/rules/flake8_bugbear/rules/function_call_argument_default.rs

@@ -29,6 +29,7 @@ use crate::rules::flake8_bugbear::rules::mutable_argument_default::is_mutable_fu
 /// def create_list() -> list[int]:
 ///     return [1, 2, 3]
 ///
+///
 /// def mutable_default(arg: list[int] = create_list()) -> list[int]:
 ///     arg.append(4)
 ///     return arg
@@ -49,6 +50,7 @@ use crate::rules::flake8_bugbear::rules::mutable_argument_default::is_mutable_fu
 /// ```python
 /// I_KNOW_THIS_IS_SHARED_STATE = create_list()
 ///
+///
 /// def mutable_default(arg: list[int] = I_KNOW_THIS_IS_SHARED_STATE) -> list[int]:
 ///     arg.append(4)
 ///     return arg

crates/ruff/src/rules/flake8_commas/rules.rs

@@ -160,9 +160,7 @@ impl AlwaysAutofixableViolation for MissingTrailingComma {
 /// import json
 ///
 ///
-/// foo = json.dumps({
-///     "bar": 1,
-/// }),
+/// foo = json.dumps({"bar": 1}),
 /// ```
 ///
 /// Use instead:
@@ -170,9 +168,7 @@ impl AlwaysAutofixableViolation for MissingTrailingComma {
 /// import json
 ///
 ///
-/// foo = json.dumps({
-///     "bar": 1,
-/// })
+/// foo = json.dumps({"bar": 1})
 /// ```
 ///
 /// In the event that a tuple is intended, then use instead:
@@ -180,11 +176,7 @@ impl AlwaysAutofixableViolation for MissingTrailingComma {
 /// import json
 ///
 ///
-/// foo = (
-///     json.dumps({
-///         "bar": 1,
-///     }),
-/// )
+/// foo = (json.dumps({"bar": 1}),)
 /// ```
 #[violation]
 pub struct TrailingCommaOnBareTuple;

crates/ruff/src/rules/flake8_django/rules/exclude_with_model_form.rs

@@ -17,6 +17,7 @@ use crate::rules::flake8_django::rules::helpers::is_model_form;
 /// ```python
 /// from django.forms import ModelForm
 ///
+///
 /// class PostForm(ModelForm):
 ///     class Meta:
 ///         model = Post
@@ -27,6 +28,7 @@ use crate::rules::flake8_django::rules::helpers::is_model_form;
 /// ```python
 /// from django.forms import ModelForm
 ///
+///
 /// class PostForm(ModelForm):
 ///     class Meta:
 ///         model = Post

crates/ruff/src/rules/flake8_django/rules/locals_in_render_function.rs

@@ -16,6 +16,7 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// from django.shortcuts import render
 ///
+///
 /// def index(request):
 ///     posts = Post.objects.all()
 ///     return render(request, "app/index.html", locals())
@@ -25,6 +26,7 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// from django.shortcuts import render
 ///
+///
 /// def index(request):
 ///     posts = Post.objects.all()
 ///     context = {"posts": posts}

crates/ruff/src/rules/flake8_type_checking/rules/runtime_import_in_type_checking_block.rs

@@ -18,6 +18,7 @@ use ruff_python_semantic::binding::{
 /// if TYPE_CHECKING:
 ///     import foo
 ///
+///
 /// def bar() -> None:
 ///     foo.bar()  # raises NameError: name 'foo' is not defined
 /// ```
@@ -26,6 +27,7 @@ use ruff_python_semantic::binding::{
 /// ```python
 /// import foo
 ///
+///
 /// def bar() -> None:
 ///     foo.bar()
 /// ```

crates/ruff/src/rules/flake8_type_checking/rules/typing_only_runtime_import.rs

@@ -130,7 +130,7 @@ impl Violation for TypingOnlyThirdPartyImport {
 ///
 /// Use instead:
 /// ```python
-/// /// from __future__ import annotations
+/// from __future__ import annotations
 ///
 /// from typing import TYPE_CHECKING
 ///

crates/ruff/src/rules/flake8_unused_arguments/rules.rs

@@ -65,6 +65,7 @@ impl Violation for UnusedFunctionArgument {
 /// ```python
 /// class MyClass:
 ///     def my_method(self, arg1):
+///         print(arg1)
 /// ```
 #[violation]
 pub struct UnusedMethodArgument {

crates/ruff/src/rules/pycodestyle/rules/compound_statements.rs

@@ -17,12 +17,12 @@ use crate::settings::{flags, Settings};
 ///
 /// ## Example
 /// ```python
-/// if foo == 'blah': do_blah_thing()
+/// if foo == "blah": do_blah_thing()
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// if foo == 'blah':
+/// if foo == "blah":
 ///     do_blah_thing()
 /// ```
 ///

crates/ruff/src/rules/pycodestyle/rules/imports.rs

@@ -43,7 +43,7 @@ impl Violation for MultipleImportsOnOneLine {
 ///
 /// ## Example
 /// ```python
-/// 'One string'
+/// "One string"
 /// "Two string"
 /// a = 1
 /// import os
@@ -54,7 +54,8 @@ impl Violation for MultipleImportsOnOneLine {
 /// ```python
 /// import os
 /// from sys import x
-/// 'One string'
+///
+/// "One string"
 /// "Two string"
 /// a = 1
 /// ```

crates/ruff/src/rules/pycodestyle/rules/invalid_escape_sequence.rs

@@ -14,12 +14,12 @@ use ruff_python_ast::source_code::Locator;
 ///
 /// ## Example
 /// ```python
-/// regex = '\.png$'
+/// regex = "\.png$"
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// regex = r'\.png$'
+/// regex = r"\.png$"
 /// ```
 #[violation]
 pub struct InvalidEscapeSequence(pub char);

crates/ruff/src/rules/pycodestyle/rules/literal_comparisons.rs

@@ -36,12 +36,15 @@ impl From<&Cmpop> for EqCmpop {
 /// ## Example
 /// ```python
 /// if arg != None:
+///     pass
 /// if None == arg:
+///     pass
 /// ```
 ///
 /// Use instead:
 /// ```python
 /// if arg is not None:
+///     pass
 /// ```
 ///
 /// ## References
@@ -78,13 +81,17 @@ impl AlwaysAutofixableViolation for NoneComparison {
 /// ## Example
 /// ```python
 /// if arg == True:
+///     pass
 /// if False == arg:
+///     pass
 /// ```
 ///
 /// Use instead:
 /// ```python
 /// if arg is True:
+///     pass
 /// if arg is False:
+///     pass
 /// ```
 ///
 /// ## References

crates/ruff/src/rules/pycodestyle/rules/not_tests.rs

@@ -16,15 +16,15 @@ use crate::rules::pycodestyle::helpers::compare;
 /// ## Example
 /// ```python
 /// Z = not X in Y
-/// if not X.B in Y:\n    pass
-///
+/// if not X.B in Y:
+///     pass
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// if x not in y:\n    pass
-/// assert (X in Y or X is Z)
-///
+/// if x not in y:
+///     pass
+/// assert X in Y or X is Z
 /// ```
 #[violation]
 pub struct NotInTest;

crates/ruff/src/rules/pycodestyle/rules/type_comparison.rs

@@ -16,12 +16,15 @@ use ruff_macros::{derive_message_formats, violation};
 /// ## Example
 /// ```python
 /// if type(obj) is type(1):
+///     pass
 /// ```
 ///
 /// Use instead:
 /// ```python
 /// if isinstance(obj, int):
+///     pass
 /// if type(a1) is type(b1):
+///     pass
 /// ```
 #[violation]
 pub struct TypeComparison;

crates/ruff/src/rules/pygrep_hooks/rules/deprecated_log_warn.rs

@@ -24,6 +24,7 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// import logging
 ///
+///
 /// def foo():
 ///     logging.warning("Something happened")
 /// ```

crates/ruff/src/rules/pylint/rules/global_statement.rs

@@ -14,11 +14,13 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// var = 1
 ///
+///
 /// def foo():
 ///     global var  # [global-statement]
 ///     var = 10
 ///     print(var)
 ///
+///
 /// foo()
 /// print(var)
 /// ```
@@ -27,10 +29,12 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// var = 1
 ///
+///
 /// def foo():
 ///     print(var)
 ///     return 10
 ///
+///
 /// var = foo()
 /// print(var)
 /// ```

crates/ruff/src/rules/pylint/rules/invalid_string_characters.rs

@@ -18,12 +18,12 @@ use ruff_python_ast::source_code::Locator;
 ///
 /// ## Example
 /// ```python
-/// x = ''
+/// x = ""
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// x = '\b'
+/// x = "\b"
 /// ```
 #[violation]
 pub struct InvalidCharacterBackspace;
@@ -51,12 +51,12 @@ impl AlwaysAutofixableViolation for InvalidCharacterBackspace {
 ///
 /// ## Example
 /// ```python
-/// x = ''
+/// x = ""
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// x = '\x1A'
+/// x = "\x1A"
 /// ```
 #[violation]
 pub struct InvalidCharacterSub;
@@ -84,12 +84,12 @@ impl AlwaysAutofixableViolation for InvalidCharacterSub {
 ///
 /// ## Example
 /// ```python
-/// x = ''
+/// x = ""
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// x = '\x1B'
+/// x = "\x1B"
 /// ```
 #[violation]
 pub struct InvalidCharacterEsc;
@@ -117,12 +117,12 @@ impl AlwaysAutofixableViolation for InvalidCharacterEsc {
 ///
 /// ## Example
 /// ```python
-/// x = ''
+/// x = ""
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// x = '\0'
+/// x = "\0"
 /// ```
 #[violation]
 pub struct InvalidCharacterNul;
@@ -149,12 +149,12 @@ impl AlwaysAutofixableViolation for InvalidCharacterNul {
 ///
 /// ## Example
 /// ```python
-/// x = 'Dear Sir/Madam'
+/// x = "Dear Sir/Madam"
 /// ```
 ///
 /// Use instead:
 /// ```python
-/// x = 'Dear Sir\u200B/\u200BMadam'  # zero width space
+/// x = "Dear Sir\u200B/\u200BMadam"  # zero width space
 /// ```
 #[violation]
 pub struct InvalidCharacterZeroWidthSpace;

crates/ruff/src/rules/ruff/rules/mutable_defaults_in_dataclass_fields.rs

@@ -25,6 +25,7 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// from dataclasses import dataclass
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = []
@@ -34,6 +35,7 @@ use crate::checkers::ast::Checker;
 /// ```python
 /// from dataclasses import dataclass, field
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = field(default_factory=list)
@@ -46,6 +48,7 @@ use crate::checkers::ast::Checker;
 ///
 /// I_KNOW_THIS_IS_SHARED_STATE = [1, 2, 3, 4]
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = I_KNOW_THIS_IS_SHARED_STATE
@@ -74,15 +77,19 @@ impl Violation for MutableDataclassDefault {
 /// ```python
 /// from dataclasses import dataclass
 ///
-/// def creating_list() -> list[]:
+///
+/// def creating_list() -> list[int]:
 ///     return [1, 2, 3, 4]
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = creating_list()
 ///
+///
 /// # also:
 ///
+///
 /// @dataclass
 /// class B:
 ///     also_mutable_default_but_sneakier: A = A()
@@ -92,13 +99,16 @@ impl Violation for MutableDataclassDefault {
 /// ```python
 /// from dataclasses import dataclass, field
 ///
-/// def creating_list() -> list[]:
+///
+/// def creating_list() -> list[int]:
 ///     return [1, 2, 3, 4]
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = field(default_factory=creating_list)
 ///
+///
 /// @dataclass
 /// class B:
 ///     also_mutable_default_but_sneakier: A = field(default_factory=A)
@@ -109,11 +119,14 @@ impl Violation for MutableDataclassDefault {
 /// ```python
 /// from dataclasses import dataclass
 ///
-/// def creating_list() -> list[]:
+///
+/// def creating_list() -> list[int]:
 ///     return [1, 2, 3, 4]
 ///
+///
 /// I_KNOW_THIS_IS_SHARED_STATE = creating_list()
 ///
+///
 /// @dataclass
 /// class A:
 ///     mutable_default: list[int] = I_KNOW_THIS_IS_SHARED_STATE

crates/ruff/src/rules/ruff/rules/unused_noqa.rs

@@ -21,6 +21,7 @@ pub struct UnusedCodes {
 /// ```python
 /// import foo  # noqa: F401
 ///
+///
 /// def bar():
 ///     foo.bar()
 /// ```
@@ -29,6 +30,7 @@ pub struct UnusedCodes {
 /// ```python
 /// import foo
 ///
+///
 /// def bar():
 ///     foo.bar()
 /// ```

docs/requirements.txt

@@ -1,3 +1,4 @@
 mkdocs~=1.4.2
 mkdocs-material~=9.0.6
 PyYAML~=6.0
+black==23.3.0

scripts/check_docs_formatted.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""Check code snippets in docs are formatted by black."""
+import argparse
+import os
+import re
+import textwrap
+from collections.abc import Sequence
+from pathlib import Path
+from re import Match
+
+import black
+from black.mode import Mode, TargetVersion
+from black.parsing import InvalidInput
+
+TARGET_VERSIONS = ["py37", "py38", "py39", "py310", "py311"]
+SNIPPED_RE = re.compile(
+    r"(?P<before>^(?P<indent> *)```\s*python\n)"
+    r"(?P<code>.*?)"
+    r"(?P<after>^(?P=indent)```\s*$)",
+    re.DOTALL | re.MULTILINE,
+)
+
+# For some rules, we don't want black to fix the formatting as this would "fix" the
+# example.
+KNOWN_FORMATTING_VIOLATIONS = [
+    "avoidable-escaped-quote",
+    "bad-quotes-docstring",
+    "bad-quotes-inline-string",
+    "bad-quotes-multiline-string",
+    "explicit-string-concatenation",
+    "line-too-long",
+    "missing-trailing-comma",
+    "multi-line-implicit-string-concatenation",
+    "multiple-statements-on-one-line-colon",
+    "multiple-statements-on-one-line-semicolon",
+    "prohibited-trailing-comma",
+    "trailing-comma-on-bare-tuple",
+    "useless-semicolon",
+]
+
+# For some docs, black is unable to parse the example code.
+KNOWN_PARSE_ERRORS = [
+    "blank-line-with-whitespace",
+    "missing-newline-at-end-of-file",
+    "mixed-spaces-and-tabs",
+    "trailing-whitespace",
+]
+
+
+class CodeBlockError(Exception):
+    """A code block parse error."""
+
+
+def format_str(
+    src: str,
+    black_mode: black.FileMode,
+) -> tuple[str, Sequence[CodeBlockError]]:
+    """Format a single docs file string."""
+    errors: list[CodeBlockError] = []
+
+    def _snipped_match(match: Match[str]) -> str:
+        code = textwrap.dedent(match["code"])
+        try:
+            code = black.format_str(code, mode=black_mode)
+        except InvalidInput as e:
+            errors.append(CodeBlockError(e))
+
+        code = textwrap.indent(code, match["indent"])
+        return f'{match["before"]}{code}{match["after"]}'
+
+    src = SNIPPED_RE.sub(_snipped_match, src)
+    return src, errors
+
+
+def format_file(
+    file: Path,
+    black_mode: black.FileMode,
+    error_known: bool,
+    args: argparse.Namespace,
+) -> int:
+    """Check the formatting of a single docs file.
+
+    Returns the exit code for the script.
+    """
+    with file.open() as f:
+        contents = f.read()
+
+    # Remove everything before the first example
+    contents = contents[contents.find("## Example") :]
+
+    # Remove everything after the last example
+    contents = contents[: contents.rfind("```")] + "```"
+
+    new_contents, errors = format_str(contents, black_mode)
+
+    if errors and not args.skip_errors and not error_known:
+        for error in errors:
+            rule_name = file.name.split(".")[0]
+            print(f"Docs parse error for `{rule_name}` docs: {error}")
+
+        return 2
+
+    if contents != new_contents:
+        rule_name = file.name.split(".")[0]
+        print(
+            f"Rule `{rule_name}` docs are not formatted. This section should be "
+            f"rewritten to:",
+        )
+
+        # Add indentation so that snipped can be copied directly to docs
+        for line in new_contents.splitlines():
+            output_line = "///"
+            if len(line) > 0:
+                output_line = f"{output_line} {line}"
+
+            print(output_line)
+
+        print("\n")
+
+        return 1
+
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Check code snippets in docs are formatted by black."""
+    parser = argparse.ArgumentParser(
+        description="Check code snippets in docs are formatted by black.",
+    )
+    parser.add_argument("--skip-errors", action="store_true")
+    parser.add_argument("--generate-docs", action="store_true")
+    args = parser.parse_args(argv)
+
+    if args.generate_docs:
+        # Generate docs
+        from generate_mkdocs import main as generate_docs
+
+        generate_docs()
+
+    # Get static docs
+    static_docs = []
+    for file in os.listdir("docs"):
+        if file.endswith(".md"):
+            static_docs.append(Path("docs") / file)
+
+    # Check rules generated
+    if not Path("docs/rules").exists():
+        print("Please generate rules first.")
+        return 1
+
+    # Get generated rules
+    generated_docs = []
+    for file in os.listdir("docs/rules"):
+        if file.endswith(".md"):
+            generated_docs.append(Path("docs/rules") / file)
+
+    if len(generated_docs) == 0:
+        print("Please generate rules first.")
+        return 1
+
+    black_mode = Mode(
+        target_versions={TargetVersion[val.upper()] for val in TARGET_VERSIONS},
+    )
+
+    # Check known formatting violations and parse errors are sorted alphabetically and
+    # have no duplicates. This will reduce the diff when adding new violations
+
+    for known_list, file_string in [
+        (KNOWN_FORMATTING_VIOLATIONS, "formatting violations"),
+        (KNOWN_PARSE_ERRORS, "parse errors"),
+    ]:
+        if known_list != sorted(known_list):
+            print(
+                f"Known {file_string} is not sorted alphabetically. Please sort and "
+                f"re-run.",
+            )
+            return 1
+
+        duplicates = list({x for x in known_list if known_list.count(x) > 1})
+        if len(duplicates) > 0:
+            print(f"Known {file_string} has duplicates:")
+            print("\n".join([f"  - {x}" for x in duplicates]))
+            print("Please remove them and re-run.")
+            return 1
+
+    violations = 0
+    errors = 0
+    for file in [*static_docs, *generated_docs]:
+        rule_name = file.name.split(".")[0]
+        if rule_name in KNOWN_FORMATTING_VIOLATIONS:
+            continue
+
+        error_known = rule_name in KNOWN_PARSE_ERRORS
+
+        result = format_file(file, black_mode, error_known, args)
+        if result == 1:
+            violations += 1
+        elif result == 2 and not error_known:
+            errors += 1
+
+    if violations > 0:
+        print(f"Formatting violations identified: {violations}")
+
+    if errors > 0:
+        print(f"New code block parse errors identified: {errors}")
+
+    if violations > 0 or errors > 0:
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/pyproject.toml

@@ -16,6 +16,7 @@ ignore = [
   "S",    # bandit
   "G",    # flake8-logging
   "T",    # flake8-print
+  "FBT",  # flake8-boolean-trap
 ]
 
 [tool.ruff.pydocstyle]