Add a formatting step using mdformat as part of generate_mkdocs.py (#10484)

## Summary

The purpose of this change is mainly to address one of the issues
outlined in #10427. Namely, some lists in the docs were not rendering
properly when preceded by a text block without a newline character. This
PR adds `mdformat` as a final step to the rule documentation script, so
that any missing newlines will be added.

NB: The default behavior of `mdformat` is to escape markdown special
characters found in text such as `<`. This resulted in some misformatted
docs. To address this I implemented an ad-hoc mdformat plugin to
override the behavior. This may be considered a bit 'hacky', but I think
it's a good solution. Nevertheless, if someone has a better idea, let me
know.

## Test Plan

This change is hard to test systematically, however, I tried my best to
look at the before and after diffs to ensure no unwanted changes were
made to the docs.

docs/requirements-insiders.txt

@@ -3,3 +3,6 @@ black==23.10.0
 mkdocs==1.5.0
 mkdocs-material @ git+ssh://git@github.com/astral-sh/mkdocs-material-insiders.git@38c0b8187325c3bab386b666daf3518ac036f2f4
 mkdocs-redirects==1.2.1
+mdformat==0.7.17
+mdformat-mkdocs==2.0.4
+mdformat-admon==2.0.2

docs/requirements.txt

@@ -3,3 +3,6 @@ black==23.10.0
 mkdocs==1.5.0
 mkdocs-material==9.1.18
 mkdocs-redirects==1.2.1
+mdformat==0.7.17
+mdformat-mkdocs==2.0.4
+mdformat-admon==2.0.2

scripts/_mdformat_utils.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import mdformat
+
+if TYPE_CHECKING:
+    import argparse
+
+    from markdown_it import MarkdownIt
+    from mdformat.renderer import RenderContext, RenderTreeNode
+
+
+class NoEscapeTextPlugin:
+    r"""Overrides the default text formatting behavior of mdformat.
+
+    By default mdformat will escape any markdown special character found in a
+    text block, e.g., <. Some of these characters are found in our
+    documentation, and when escaped (i.e. \<) will be rendered incorrectly by
+    mkdocs, i.e., the backslash will appear in the render. Because our only
+    purpose in using mdformat is to manage the line-breaks, it makes sense to
+    override its text formatting behavior.
+    """
+
+    def __init__(self: NoEscapeTextPlugin) -> None:
+        self.POSTPROCESSORS = {"text": NoEscapeTextPlugin.text}
+        self.RENDERERS = {}
+
+    @staticmethod
+    def add_cli_options(parser: argparse.ArgumentParser) -> None:
+        pass
+
+    @staticmethod
+    def update_mdit(mdit: MarkdownIt) -> None:
+        pass
+
+    @staticmethod
+    def text(_text: str, node: RenderTreeNode, _context: RenderContext) -> str:
+        return node.content
+
+
+def add_no_escape_text_plugin() -> None:
+    """Add NoEscapeTextPlugin to the list of mdformat extensions."""
+    mdformat.plugins.PARSER_EXTENSIONS["no-escape-text"] = NoEscapeTextPlugin()

scripts/generate_mkdocs.py

@@ -9,8 +9,11 @@ import subprocess
 from pathlib import Path
 from typing import NamedTuple
 
+import mdformat
 import yaml
 
+from _mdformat_utils import add_no_escape_text_plugin
+
 
 class Section(NamedTuple):
     """A section to include in the MkDocs documentation."""
@@ -140,6 +143,11 @@ def main() -> None:
 
             f.write(clean_file_content(file_content, title))
 
+    # Format rules docs
+    add_no_escape_text_plugin()
+    for rule_doc in Path("docs/rules").glob("*.md"):
+        mdformat.file(rule_doc, extensions=["mkdocs", "admonition", "no-escape-text"])
+
     with Path("mkdocs.template.yml").open(encoding="utf8") as fp:
         config = yaml.safe_load(fp)