GH-137623: Begin enforcing docstring length in Argument Clinic (#137624)

Lib/test/clinic.test.c

@@ -5084,14 +5084,18 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg)
 Test.__init__
     *args: tuple
 
-Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE.
+Varargs init method.
+
+For example, nargs is translated to PyTuple_GET_SIZE.
 [clinic start generated code]*/
 
 PyDoc_STRVAR(Test___init____doc__,
 "Test(*args)\n"
 "--\n"
 "\n"
-"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE.");
+"Varargs init method.\n"
+"\n"
+"For example, nargs is translated to PyTuple_GET_SIZE.");
 
 static int
 Test___init___impl(TestObj *self, PyObject *args);
@@ -5120,21 +5124,25 @@ exit:
 
 static int
 Test___init___impl(TestObj *self, PyObject *args)
-/*[clinic end generated code: output=f172425cec373cd6 input=4b8388c4e6baab6f]*/
+/*[clinic end generated code: output=0e5836c40dbc2397 input=a615a4485c0fc3e2]*/
 
 /*[clinic input]
 @classmethod
 Test.__new__
     *args: tuple
 
-Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE.
+Varargs new method.
+
+For example, nargs is translated to PyTuple_GET_SIZE.
 [clinic start generated code]*/
 
 PyDoc_STRVAR(Test__doc__,
 "Test(*args)\n"
 "--\n"
 "\n"
-"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE.");
+"Varargs new method.\n"
+"\n"
+"For example, nargs is translated to PyTuple_GET_SIZE.");
 
 static PyObject *
 Test_impl(PyTypeObject *type, PyObject *args);
@@ -5162,7 +5170,7 @@ exit:
 
 static PyObject *
 Test_impl(PyTypeObject *type, PyObject *args)
-/*[clinic end generated code: output=ee1e8892a67abd4a input=a8259521129cad20]*/
+/*[clinic end generated code: output=e6fba0c8951882fd input=8ce30adb836aeacb]*/
 
 
 /*[clinic input]

Modules/clinic/posixmodule.c.h

@@ -215,8 +215,8 @@ PyDoc_STRVAR(os_access__doc__,
 "  NotImplementedError.\n"
 "\n"
 "Note that most operations will use the effective uid/gid, therefore this\n"
-"  routine can be used in a suid/sgid environment to test if the invoking user\n"
-"  has the specified access to the path.");
+"  routine can be used in a suid/sgid environment to test if the invoking\n"
+"  user has the specified access to the path.");
 
 #define OS_ACCESS_METHODDEF    \
     {"access", _PyCFunction_CAST(os_access), METH_FASTCALL|METH_KEYWORDS, os_access__doc__},
@@ -13419,4 +13419,4 @@ exit:
 #ifndef OS__EMSCRIPTEN_LOG_METHODDEF
     #define OS__EMSCRIPTEN_LOG_METHODDEF
 #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */
-/*[clinic end generated code: output=b1e2615384347102 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=23de5d098e2dd73f input=a9049054013a1b77]*/

Modules/posixmodule.c

@@ -3295,15 +3295,15 @@ dir_fd, effective_ids, and follow_symlinks may not be implemented
   NotImplementedError.
 
 Note that most operations will use the effective uid/gid, therefore this
-  routine can be used in a suid/sgid environment to test if the invoking user
-  has the specified access to the path.
+  routine can be used in a suid/sgid environment to test if the invoking
+  user has the specified access to the path.
 
 [clinic start generated code]*/
 
 static int
 os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd,
                int effective_ids, int follow_symlinks)
-/*[clinic end generated code: output=cf84158bc90b1a77 input=3ffe4e650ee3bf20]*/
+/*[clinic end generated code: output=cf84158bc90b1a77 input=c33565f7584b99e4]*/
 {
     int return_value;
 

Tools/clinic/libclinic/_overlong_docstrings.py

@@ -0,0 +1,299 @@
+OVERLONG_SUMMARY = frozenset((
+    # Lib/test/
+    'test_preprocessor_guarded_if_e_or_f',
+
+    # Modules/
+    '_abc._abc_init',
+    '_abc._abc_instancecheck',
+    '_abc._abc_register',
+    '_abc._abc_subclasscheck',
+    '_codecs.lookup',
+    '_ctypes.byref',
+    '_curses.can_change_color',
+    '_curses.is_term_resized',
+    '_curses.mousemask',
+    '_curses.reset_prog_mode',
+    '_curses.reset_shell_mode',
+    '_curses.termname',
+    '_curses.window.enclose',
+    '_functools.reduce',
+    '_gdbm.gdbm.setdefault',
+    '_hashlib.HMAC.hexdigest',
+    '_hashlib.openssl_shake_128',
+    '_hashlib.openssl_shake_256',
+    '_hashlib.pbkdf2_hmac',
+    '_hmac.HMAC.hexdigest',
+    '_interpreters.is_shareable',
+    '_io._BufferedIOBase.read1',
+    '_lzma._decode_filter_properties',
+    '_remote_debugging.RemoteUnwinder.__init__',
+    '_remote_debugging.RemoteUnwinder.get_all_awaited_by',
+    '_remote_debugging.RemoteUnwinder.get_async_stack_trace',
+    '_socket.inet_aton',
+    '_sre.SRE_Match.expand',
+    '_sre.SRE_Match.groupdict',
+    '_sre.SRE_Pattern.finditer',
+    '_sre.SRE_Pattern.search',
+    '_sre.SRE_Pattern.sub',
+    '_sre.SRE_Pattern.subn',
+    '_ssl._SSLContext.sni_callback',
+    '_ssl._SSLSocket.pending',
+    '_ssl._SSLSocket.sendfile',
+    '_ssl.get_default_verify_paths',
+    '_ssl.RAND_status',
+    '_sysconfig.config_vars',
+    '_testcapi.make_exception_with_doc',
+    '_testcapi.VectorCallClass.set_vectorcall',
+    '_tkinter.getbusywaitinterval',
+    '_tkinter.setbusywaitinterval',
+    '_tracemalloc.reset_peak',
+    '_zstd.get_frame_size',
+    '_zstd.set_parameter_types',
+    '_zstd.ZstdDecompressor.decompress',
+    'array.array.buffer_info',
+    'array.array.frombytes',
+    'array.array.fromfile',
+    'array.array.tobytes',
+    'cmath.isfinite',
+    'datetime.datetime.strptime',
+    'gc.get_objects',
+    'itertools.chain.from_iterable',
+    'itertools.combinations_with_replacement.__new__',
+    'itertools.cycle.__new__',
+    'itertools.starmap.__new__',
+    'itertools.takewhile.__new__',
+    'math.comb',
+    'math.perm',
+    'os.getresgid',
+    'os.lstat',
+    'os.pread',
+    'os.pwritev',
+    'os.sched_getaffinity',
+    'os.sched_rr_get_interval',
+    'os.timerfd_gettime',
+    'os.timerfd_gettime_ns',
+    'os.urandom',
+    'os.WIFEXITED',
+    'os.WTERMSIG',
+    'pwd.getpwall',
+    'pyexpat.xmlparser.ExternalEntityParserCreate',
+    'pyexpat.xmlparser.GetReparseDeferralEnabled',
+    'pyexpat.xmlparser.SetParamEntityParsing',
+    'pyexpat.xmlparser.UseForeignDTD',
+    'readline.redisplay',
+    'signal.set_wakeup_fd',
+    'unicodedata.UCD.combining',
+    'unicodedata.UCD.decomposition',
+    'zoneinfo.ZoneInfo.dst',
+    'zoneinfo.ZoneInfo.tzname',
+    'zoneinfo.ZoneInfo.utcoffset',
+
+    # Objects/
+    'B.zfill',
+    'bytearray.count',
+    'bytearray.endswith',
+    'bytearray.extend',
+    'bytearray.find',
+    'bytearray.index',
+    'bytearray.maketrans',
+    'bytearray.rfind',
+    'bytearray.rindex',
+    'bytearray.rsplit',
+    'bytearray.split',
+    'bytearray.splitlines',
+    'bytearray.startswith',
+    'bytes.count',
+    'bytes.endswith',
+    'bytes.find',
+    'bytes.index',
+    'bytes.maketrans',
+    'bytes.rfind',
+    'bytes.rindex',
+    'bytes.startswith',
+    'code.replace',
+    'complex.conjugate',
+    'dict.pop',
+    'float.as_integer_ratio',
+    'frame.f_trace',
+    'int.bit_count',
+    'OrderedDict.fromkeys',
+    'OrderedDict.pop',
+    'set.symmetric_difference_update',
+    'str.count',
+    'str.endswith',
+    'str.find',
+    'str.index',
+    'str.isprintable',
+    'str.rfind',
+    'str.rindex',
+    'str.rsplit',
+    'str.split',
+    'str.startswith',
+    'str.strip',
+    'str.swapcase',
+    'str.zfill',
+
+    # PC/
+    'msvcrt.kbhit',
+
+    # Python/
+    '_jit.is_active',
+    '_jit.is_available',
+    '_jit.is_enabled',
+    'marshal.dumps',
+    'sys._current_exceptions',
+    'sys._setprofileallthreads',
+    'sys._settraceallthreads',
+))
+
+OVERLONG_BODY = frozenset((
+    # Modules/
+    '_bz2.BZ2Decompressor.decompress',
+    '_curses.color_content',
+    '_curses.flash',
+    '_curses.longname',
+    '_curses.resize_term',
+    '_curses.use_env',
+    '_curses.window.border',
+    '_curses.window.derwin',
+    '_curses.window.getch',
+    '_curses.window.getkey',
+    '_curses.window.inch',
+    '_curses.window.insch',
+    '_curses.window.insnstr',
+    '_curses.window.is_linetouched',
+    '_curses.window.noutrefresh',
+    '_curses.window.overlay',
+    '_curses.window.overwrite',
+    '_curses.window.refresh',
+    '_curses.window.scroll',
+    '_curses.window.subwin',
+    '_curses.window.touchline',
+    '_curses_panel.panel.hide',
+    '_functools.reduce',
+    '_hashlib.HMAC.hexdigest',
+    '_hmac.HMAC.hexdigest',
+    '_interpreters.capture_exception',
+    '_io._IOBase.seek',
+    '_io._TextIOBase.detach',
+    '_io.FileIO.read',
+    '_io.FileIO.readall',
+    '_io.FileIO.seek',
+    '_io.open',
+    '_io.open_code',
+    '_lzma.LZMADecompressor.decompress',
+    '_multibytecodec.MultibyteCodec.decode',
+    '_multibytecodec.MultibyteCodec.encode',
+    '_posixsubprocess.fork_exec',
+    '_remote_debugging.RemoteUnwinder.__init__',
+    '_remote_debugging.RemoteUnwinder.get_all_awaited_by',
+    '_remote_debugging.RemoteUnwinder.get_async_stack_trace',
+    '_remote_debugging.RemoteUnwinder.get_stack_trace',
+    '_socket.socket.send',
+    '_sqlite3.Blob.read',
+    '_sqlite3.Blob.seek',
+    '_sqlite3.Blob.write',
+    '_sqlite3.Connection.deserialize',
+    '_sqlite3.Connection.serialize',
+    '_sqlite3.Connection.set_progress_handler',
+    '_sqlite3.Connection.setlimit',
+    '_ssl._SSLContext.sni_callback',
+    '_ssl._SSLSocket.context',
+    '_ssl._SSLSocket.get_channel_binding',
+    '_ssl._SSLSocket.sendfile',
+    '_tkinter.setbusywaitinterval',
+    '_zstd.ZstdCompressor.compress',
+    '_zstd.ZstdCompressor.flush',
+    '_zstd.ZstdCompressor.set_pledged_input_size',
+    '_zstd.ZstdDecompressor.__new__',
+    '_zstd.ZstdDecompressor.decompress',
+    '_zstd.ZstdDecompressor.unused_data',
+    '_zstd.ZstdDict.__new__',
+    '_zstd.ZstdDict.as_digested_dict',
+    '_zstd.ZstdDict.as_prefix',
+    '_zstd.ZstdDict.as_undigested_dict',
+    'array.array.byteswap',
+    'array.array.fromunicode',
+    'array.array.tounicode',
+    'binascii.a2b_base64',
+    'cmath.isclose',
+    'datetime.date.fromtimestamp',
+    'datetime.datetime.fromtimestamp',
+    'datetime.time.strftime',
+    'fcntl.ioctl',
+    'fcntl.lockf',
+    'gc.freeze',
+    'itertools.combinations_with_replacement.__new__',
+    'math.nextafter',
+    'os.fspath',
+    'os.link',
+    'os.listdir',
+    'os.listxattr',
+    'os.lseek',
+    'os.mknod',
+    'os.preadv',
+    'os.pwritev',
+    'os.readinto',
+    'os.rename',
+    'os.replace',
+    'os.setxattr',
+    'pyexpat.xmlparser.GetInputContext',
+    'pyexpat.xmlparser.UseForeignDTD',
+    'select.devpoll',
+    'select.poll',
+    'select.select',
+    'signal.setitimer',
+    'signal.signal',
+    'termios.tcsetwinsize',
+    'zlib.Decompress.decompress',
+    'zlib.ZlibDecompressor.decompress',
+
+    # Objects/
+    'bytearray.maketrans',
+    'bytearray.partition',
+    'bytearray.replace',
+    'bytearray.rpartition',
+    'bytearray.rsplit',
+    'bytearray.splitlines',
+    'bytearray.strip',
+    'bytes.maketrans',
+    'bytes.partition',
+    'bytes.replace',
+    'bytes.rpartition',
+    'bytes.rsplit',
+    'bytes.splitlines',
+    'bytes.strip',
+    'float.__getformat__',
+    'list.sort',
+    'memoryview.tobytes',
+    'str.capitalize',
+    'str.isalnum',
+    'str.isalpha',
+    'str.isdecimal',
+    'str.isdigit',
+    'str.isidentifier',
+    'str.islower',
+    'str.isnumeric',
+    'str.isspace',
+    'str.isupper',
+    'str.join',
+    'str.partition',
+    'str.removeprefix',
+    'str.replace',
+    'str.rpartition',
+    'str.splitlines',
+    'str.title',
+    'str.translate',
+
+    # PC/
+    '_wmi.exec_query',
+
+    # Python/
+    '__import__',
+    '_contextvars.ContextVar.get',
+    '_contextvars.ContextVar.reset',
+    '_contextvars.ContextVar.set',
+    '_imp.acquire_lock',
+    'marshal.dumps',
+    'sys._stats_dump',
+))

Tools/clinic/libclinic/dsl_parser.py

@@ -14,6 +14,7 @@ import libclinic
 from libclinic import (
     ClinicError, VersionTuple,
     fail, warn, unspecified, unknown, NULL)
+from libclinic._overlong_docstrings import OVERLONG_SUMMARY, OVERLONG_BODY
 from libclinic.function import (
     Module, Class, Function, Parameter,
     FunctionKind,
@@ -1515,6 +1516,28 @@ class DSLParser:
             # between it and the {parameters} we're about to add.
             lines.append('')
 
+        # Fail if the summary line is too long.
+        # Warn if any of the body lines are too long.
+        # Existing violations are recorded in OVERLONG_{SUMMARY,BODY}.
+        max_width = f.docstring_line_width
+        summary_len = len(lines[0])
+        max_body = max(map(len, lines[1:]))
+        if summary_len > max_width:
+            if f.full_name not in OVERLONG_SUMMARY:
+                fail(f"Summary line for {f.full_name!r} is too long!\n"
+                     f"The summary line must be no longer than {max_width} characters.")
+        else:
+            if f.full_name in OVERLONG_SUMMARY:
+                warn(f"Remove {f.full_name!r} from OVERLONG_SUMMARY!\n")
+
+        if max_body > max_width:
+            if f.full_name not in OVERLONG_BODY:
+                warn(f"Docstring lines for {f.full_name!r} are too long!\n"
+                     f"Lines should be no longer than {max_width} characters.")
+        else:
+            if f.full_name in OVERLONG_BODY:
+                warn(f"Remove {f.full_name!r} from OVERLONG_BODY!\n")
+
         parameters_marker_count = len(f.docstring.split('{parameters}')) - 1
         if parameters_marker_count > 1:
             fail('You may not specify {parameters} more than once in a docstring!')

Tools/clinic/libclinic/function.py

@@ -167,6 +167,19 @@ class Function:
             flags.append('METH_COEXIST')
         return '|'.join(flags)
 
+    @property
+    def docstring_line_width(self) -> int:
+        """Return the maximum line width for docstring lines.
+
+        Pydoc adds indentation when displaying functions and methods.
+        To keep the total width of within 80 characters, we use a
+        maximum of 76 characters for global functions and classes,
+        and 72 characters for methods.
+        """
+        if self.cls is not None and not self.kind.new_or_init:
+            return 72
+        return 76
+
     def __repr__(self) -> str:
         return f'<clinic.Function {self.name!r}>'