GH-121970: Extract pydoc_topics into a new extension (#131256)

Doc/conf.py

@@ -33,6 +33,7 @@ extensions = [
     'issue_role',
     'lexers',
     'misc_news',
+    'pydoc_topics',
     'pyspecific',
     'sphinx.ext.coverage',
     'sphinx.ext.doctest',

Doc/tools/extensions/pydoc_topics.py

@@ -0,0 +1,188 @@
+"""Support for building "topic help" for pydoc."""
+
+from __future__ import annotations
+
+from time import asctime
+from typing import TYPE_CHECKING
+
+from sphinx.builders.text import TextBuilder
+from sphinx.util import logging
+from sphinx.util.display import status_iterator
+from sphinx.util.docutils import new_document
+from sphinx.writers.text import TextTranslator
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence, Set
+
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+logger = logging.getLogger(__name__)
+
+_PYDOC_TOPIC_LABELS: Sequence[str] = sorted({
+    "assert",
+    "assignment",
+    "assignment-expressions",
+    "async",
+    "atom-identifiers",
+    "atom-literals",
+    "attribute-access",
+    "attribute-references",
+    "augassign",
+    "await",
+    "binary",
+    "bitwise",
+    "bltin-code-objects",
+    "bltin-ellipsis-object",
+    "bltin-null-object",
+    "bltin-type-objects",
+    "booleans",
+    "break",
+    "callable-types",
+    "calls",
+    "class",
+    "comparisons",
+    "compound",
+    "context-managers",
+    "continue",
+    "conversions",
+    "customization",
+    "debugger",
+    "del",
+    "dict",
+    "dynamic-features",
+    "else",
+    "exceptions",
+    "execmodel",
+    "exprlists",
+    "floating",
+    "for",
+    "formatstrings",
+    "function",
+    "global",
+    "id-classes",
+    "identifiers",
+    "if",
+    "imaginary",
+    "import",
+    "in",
+    "integers",
+    "lambda",
+    "lists",
+    "naming",
+    "nonlocal",
+    "numbers",
+    "numeric-types",
+    "objects",
+    "operator-summary",
+    "pass",
+    "power",
+    "raise",
+    "return",
+    "sequence-types",
+    "shifting",
+    "slicings",
+    "specialattrs",
+    "specialnames",
+    "string-methods",
+    "strings",
+    "subscriptions",
+    "truth",
+    "try",
+    "types",
+    "typesfunctions",
+    "typesmapping",
+    "typesmethods",
+    "typesmodules",
+    "typesseq",
+    "typesseq-mutable",
+    "unary",
+    "while",
+    "with",
+    "yield",
+})
+
+
+class PydocTopicsBuilder(TextBuilder):
+    name = "pydoc-topics"
+
+    def init(self) -> None:
+        super().init()
+        self.topics: dict[str, str] = {}
+
+    def get_outdated_docs(self) -> str:
+        # Return a string describing what an update build will build.
+        return "all pydoc topics"
+
+    def write_documents(self, _docnames: Set[str]) -> None:
+        env = self.env
+
+        labels: dict[str, tuple[str, str, str]]
+        labels = env.domains.standard_domain.labels
+
+        # docname -> list of (topic_label, label_id) pairs
+        doc_labels: dict[str, list[tuple[str, str]]] = {}
+        for topic_label in _PYDOC_TOPIC_LABELS:
+            try:
+                docname, label_id, _section_name = labels[topic_label]
+            except KeyError:
+                logger.warning("label %r not in documentation", topic_label)
+                continue
+            doc_labels.setdefault(docname, []).append((topic_label, label_id))
+
+        for docname, label_ids in status_iterator(
+            doc_labels.items(),
+            "building topics... ",
+            length=len(doc_labels),
+            stringify_func=_display_labels,
+        ):
+            doctree = env.get_and_resolve_doctree(docname, builder=self)
+            doc_ids = doctree.ids
+            for topic_label, label_id in label_ids:
+                document = new_document("<section node>")
+                document.append(doc_ids[label_id])
+                visitor = TextTranslator(document, builder=self)
+                document.walkabout(visitor)
+                body = "\n".join(map(str.rstrip, visitor.body.splitlines()))
+                self.topics[topic_label] = body + "\n"
+
+    def finish(self) -> None:
+        topics_repr = "\n".join(
+            f"    '{topic}': {_repr(self.topics[topic])},"
+            for topic in sorted(self.topics)
+        )
+        topics = f"""\
+# Autogenerated by Sphinx on {asctime()}
+# as part of the release process.
+
+topics = {{
+{topics_repr}
+}}
+"""
+        self.outdir.joinpath("topics.py").write_text(topics, encoding="utf-8")
+
+
+def _display_labels(item: tuple[str, Sequence[tuple[str, str]]]) -> str:
+    _docname, label_ids = item
+    labels = [name for name, _id in label_ids]
+    if len(labels) > 4:
+        return f"{labels[0]}, {labels[1]}, ..., {labels[-2]}, {labels[-1]}"
+    return ", ".join(labels)
+
+
+def _repr(text: str, /) -> str:
+    """Return a triple-single-quoted representation of text."""
+    if "'''" not in text:
+        return f"r'''{text}'''"
+    text = text.replace("\\", "\\\\").replace("'''", r"\'\'\'")
+    return f"'''{text}'''"
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_builder(PydocTopicsBuilder)
+
+    return {
+        "version": "1.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

Doc/tools/extensions/pyspecific.py

@@ -12,20 +12,14 @@
 import re
 import io
 from os import getenv, path
-from time import asctime
-from pprint import pformat
 
 from docutils import nodes
-from docutils.io import StringOutput
 from docutils.parsers.rst import directives
-from docutils.utils import new_document, unescape
+from docutils.utils import unescape
 from sphinx import addnodes
-from sphinx.builders import Builder
 from sphinx.domains.python import PyFunction, PyMethod, PyModule
 from sphinx.locale import _ as sphinx_gettext
 from sphinx.util.docutils import SphinxDirective
-from sphinx.writers.text import TextWriter, TextTranslator
-from sphinx.util.display import status_iterator
 
 # Used in conf.py and updated here by python/release-tools/run_release.py
 SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s'
@@ -57,69 +51,6 @@ class PyAwaitableMethod(PyAwaitableMixin, PyMethod):
         return PyMethod.run(self)
 
 
-# Support for building "topic help" for pydoc
-
-pydoc_topic_labels = [
-    'assert', 'assignment', 'assignment-expressions', 'async',  'atom-identifiers',
-    'atom-literals', 'attribute-access', 'attribute-references', 'augassign', 'await',
-    'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
-    'bltin-null-object', 'bltin-type-objects', 'booleans',
-    'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound',
-    'context-managers', 'continue', 'conversions', 'customization', 'debugger',
-    'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel',
-    'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global',
-    'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers',
-    'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types',
-    'objects', 'operator-summary', 'pass', 'power', 'raise', 'return',
-    'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames',
-    'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types',
-    'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules',
-    'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield'
-]
-
-
-class PydocTopicsBuilder(Builder):
-    name = 'pydoc-topics'
-
-    default_translator_class = TextTranslator
-
-    def init(self):
-        self.topics = {}
-        self.secnumbers = {}
-
-    def get_outdated_docs(self):
-        return 'all pydoc topics'
-
-    def get_target_uri(self, docname, typ=None):
-        return ''  # no URIs
-
-    def write(self, *ignored):
-        writer = TextWriter(self)
-        for label in status_iterator(pydoc_topic_labels,
-                                     'building topics... ',
-                                     length=len(pydoc_topic_labels)):
-            if label not in self.env.domaindata['std']['labels']:
-                self.env.logger.warning(f'label {label!r} not in documentation')
-                continue
-            docname, labelid, sectname = self.env.domaindata['std']['labels'][label]
-            doctree = self.env.get_and_resolve_doctree(docname, self)
-            document = new_document('<section node>')
-            document.append(doctree.ids[labelid])
-            destination = StringOutput(encoding='utf-8')
-            writer.write(document, destination)
-            self.topics[label] = writer.output
-
-    def finish(self):
-        f = open(path.join(self.outdir, 'topics.py'), 'wb')
-        try:
-            f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8'))
-            f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8'))
-            f.write('# as part of the release process.\n'.encode('utf-8'))
-            f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8'))
-        finally:
-            f.close()
-
-
 # Support for documenting Opcodes
 
 opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?')
@@ -196,7 +127,6 @@ def patch_pairindextypes(app, _env) -> None:
 
 
 def setup(app):
-    app.add_builder(PydocTopicsBuilder)
     app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
     app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command)
     app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event)