bpo-43795: Generate python3dll.c and doc data from manifest (PEP 652) (GH-25315)

Doc/data/stable_abi.dat

@@ -1,5 +1,13 @@
-# File generated by 'make regen-limited-abi'
+# Generated by Tools/scripts/stable_abi.py
-# This is NOT an authoritative list of stable ABI symbols
+
+METH_CLASS
+METH_COEXIST
+METH_FASTCALL
+METH_METHOD
+METH_NOARGS
+METH_O
+METH_STATIC
+METH_VARARGS
 PyAiter_Check
 PyArg_Parse
 PyArg_ParseTuple
@@ -33,6 +41,8 @@ PyBytes_FromStringAndSize
 PyBytes_Repr
 PyBytes_Size
 PyBytes_Type
+PyCFunction
+PyCFunctionWithKeywords
 PyCFunction_Call
 PyCFunction_GetFlags
 PyCFunction_GetFunction
@@ -44,6 +54,7 @@ PyCMethod_New
 PyCallIter_New
 PyCallIter_Type
 PyCallable_Check
+PyCapsule_Destructor
 PyCapsule_GetContext
 PyCapsule_GetDestructor
 PyCapsule_GetName
@@ -138,11 +149,17 @@ PyErr_PrintEx
 PyErr_ProgramText
 PyErr_ResourceWarning
 PyErr_Restore
+PyErr_SetExcFromWindowsErr
+PyErr_SetExcFromWindowsErrWithFilename
+PyErr_SetExcFromWindowsErrWithFilenameObject
+PyErr_SetExcFromWindowsErrWithFilenameObjects
 PyErr_SetExcInfo
 PyErr_SetFromErrno
 PyErr_SetFromErrnoWithFilename
 PyErr_SetFromErrnoWithFilenameObject
 PyErr_SetFromErrnoWithFilenameObjects
+PyErr_SetFromWindowsErr
+PyErr_SetFromWindowsErrWithFilename
 PyErr_SetImportError
 PyErr_SetImportErrorSubclass
 PyErr_SetInterrupt
@@ -243,6 +260,7 @@ PyExc_UnicodeWarning
 PyExc_UserWarning
 PyExc_ValueError
 PyExc_Warning
+PyExc_WindowsError
 PyExc_ZeroDivisionError
 PyExceptionClass_Name
 PyException_GetCause
@@ -274,6 +292,8 @@ PyGC_IsEnabled
 PyGILState_Ensure
 PyGILState_GetThisThreadState
 PyGILState_Release
+PyGILState_STATE
+PyGetSetDef
 PyGetSetDescr_Type
 PyImport_AddModule
 PyImport_AddModuleObject
@@ -296,6 +316,7 @@ PyImport_ImportModuleLevelObject
 PyImport_ImportModuleNoBlock
 PyImport_ReloadModule
 PyIndex_Check
+PyInterpreterState
 PyInterpreterState_Clear
 PyInterpreterState_Delete
 PyInterpreterState_Get
@@ -319,6 +340,7 @@ PyList_SetSlice
 PyList_Size
 PyList_Sort
 PyList_Type
+PyLongObject
 PyLongRangeIter_Type
 PyLong_AsDouble
 PyLong_AsLong
@@ -358,12 +380,16 @@ PyMem_Calloc
 PyMem_Free
 PyMem_Malloc
 PyMem_Realloc
+PyMemberDef
 PyMemberDescr_Type
 PyMemoryView_FromMemory
 PyMemoryView_FromObject
 PyMemoryView_GetContiguous
 PyMemoryView_Type
+PyMethodDef
 PyMethodDescr_Type
+PyModuleDef
+PyModuleDef_Base
 PyModuleDef_Init
 PyModuleDef_Type
 PyModule_AddFunctions
@@ -427,18 +453,22 @@ PyOS_AfterFork
 PyOS_AfterFork_Child
 PyOS_AfterFork_Parent
 PyOS_BeforeFork
+PyOS_CheckStack
 PyOS_FSPath
+PyOS_InputHook
 PyOS_InterruptOccurred
 PyOS_double_to_string
 PyOS_getsig
 PyOS_mystricmp
 PyOS_mystrnicmp
 PyOS_setsig
+PyOS_sighandler_t
 PyOS_snprintf
 PyOS_string_to_double
 PyOS_strtol
 PyOS_strtoul
 PyOS_vsnprintf
+PyObject
 PyObject_ASCII
 PyObject_AsFileDescriptor
 PyObject_Bytes
@@ -537,6 +567,8 @@ PySlice_Unpack
 PyState_AddModule
 PyState_FindModule
 PyState_RemoveModule
+PyStructSequence_Desc
+PyStructSequence_Field
 PyStructSequence_GetItem
 PyStructSequence_New
 PyStructSequence_NewType
@@ -557,6 +589,7 @@ PySys_SetObject
 PySys_SetPath
 PySys_WriteStderr
 PySys_WriteStdout
+PyThreadState
 PyThreadState_Clear
 PyThreadState_Delete
 PyThreadState_Get
@@ -604,6 +637,7 @@ PyTuple_Pack
 PyTuple_SetItem
 PyTuple_Size
 PyTuple_Type
+PyTypeObject
 PyType_ClearCache
 PyType_FromModuleAndSpec
 PyType_FromSpec
@@ -617,6 +651,8 @@ PyType_GetSlot
 PyType_IsSubtype
 PyType_Modified
 PyType_Ready
+PyType_Slot
+PyType_Spec
 PyType_Type
 PyUnicodeDecodeError_Create
 PyUnicodeDecodeError_GetEncoding
@@ -653,6 +689,7 @@ PyUnicode_AsEncodedObject
 PyUnicode_AsEncodedString
 PyUnicode_AsEncodedUnicode
 PyUnicode_AsLatin1String
+PyUnicode_AsMBCSString
 PyUnicode_AsRawUnicodeEscapeString
 PyUnicode_AsUCS4
 PyUnicode_AsUCS4Copy
@@ -672,11 +709,14 @@ PyUnicode_Count
 PyUnicode_Decode
 PyUnicode_DecodeASCII
 PyUnicode_DecodeCharmap
+PyUnicode_DecodeCodePageStateful
 PyUnicode_DecodeFSDefault
 PyUnicode_DecodeFSDefaultAndSize
 PyUnicode_DecodeLatin1
 PyUnicode_DecodeLocale
 PyUnicode_DecodeLocaleAndSize
+PyUnicode_DecodeMBCS
+PyUnicode_DecodeMBCSStateful
 PyUnicode_DecodeRawUnicodeEscape
 PyUnicode_DecodeUTF16
 PyUnicode_DecodeUTF16Stateful
@@ -687,6 +727,7 @@ PyUnicode_DecodeUTF7Stateful
 PyUnicode_DecodeUTF8
 PyUnicode_DecodeUTF8Stateful
 PyUnicode_DecodeUnicodeEscape
+PyUnicode_EncodeCodePage
 PyUnicode_EncodeFSDefault
 PyUnicode_EncodeLocale
 PyUnicode_FSConverter
@@ -724,6 +765,8 @@ PyUnicode_Tailmatch
 PyUnicode_Translate
 PyUnicode_Type
 PyUnicode_WriteChar
+PyVarObject
+PyWeakReference
 PyWeakref_GetObject
 PyWeakref_NewProxy
 PyWeakref_NewRef
@@ -732,11 +775,14 @@ PyWrapper_New
 PyZip_Type
 Py_AddPendingCall
 Py_AtExit
+Py_BEGIN_ALLOW_THREADS
+Py_BLOCK_THREADS
 Py_BuildValue
 Py_BytesMain
 Py_CompileString
 Py_DecRef
 Py_DecodeLocale
+Py_END_ALLOW_THREADS
 Py_EncodeLocale
 Py_EndInterpreter
 Py_EnterRecursiveCall
@@ -781,6 +827,127 @@ Py_SetPath
 Py_SetProgramName
 Py_SetPythonHome
 Py_SetRecursionLimit
+Py_TPFLAGS_BASETYPE
+Py_TPFLAGS_DEFAULT
+Py_TPFLAGS_HAVE_GC
+Py_TPFLAGS_METHOD_DESCRIPTOR
+Py_UCS4
+Py_UNBLOCK_THREADS
 Py_UTF8Mode
 Py_VaBuildValue
 Py_XNewRef
+Py_am_aiter
+Py_am_anext
+Py_am_await
+Py_am_send
+Py_intptr_t
+Py_mp_ass_subscript
+Py_mp_length
+Py_mp_subscript
+Py_nb_absolute
+Py_nb_add
+Py_nb_and
+Py_nb_bool
+Py_nb_divmod
+Py_nb_float
+Py_nb_floor_divide
+Py_nb_index
+Py_nb_inplace_add
+Py_nb_inplace_and
+Py_nb_inplace_floor_divide
+Py_nb_inplace_lshift
+Py_nb_inplace_matrix_multiply
+Py_nb_inplace_multiply
+Py_nb_inplace_or
+Py_nb_inplace_power
+Py_nb_inplace_remainder
+Py_nb_inplace_rshift
+Py_nb_inplace_subtract
+Py_nb_inplace_true_divide
+Py_nb_inplace_xor
+Py_nb_int
+Py_nb_invert
+Py_nb_lshift
+Py_nb_matrix_multiply
+Py_nb_multiply
+Py_nb_negative
+Py_nb_or
+Py_nb_positive
+Py_nb_power
+Py_nb_remainder
+Py_nb_rshift
+Py_nb_subtract
+Py_nb_true_divide
+Py_nb_xor
+Py_sq_ass_item
+Py_sq_concat
+Py_sq_contains
+Py_sq_inplace_concat
+Py_sq_inplace_repeat
+Py_sq_item
+Py_sq_length
+Py_sq_repeat
+Py_ssize_t
+Py_tp_alloc
+Py_tp_base
+Py_tp_bases
+Py_tp_call
+Py_tp_clear
+Py_tp_dealloc
+Py_tp_del
+Py_tp_descr_get
+Py_tp_descr_set
+Py_tp_doc
+Py_tp_finalize
+Py_tp_free
+Py_tp_getattr
+Py_tp_getattro
+Py_tp_getset
+Py_tp_hash
+Py_tp_init
+Py_tp_is_gc
+Py_tp_iter
+Py_tp_iternext
+Py_tp_members
+Py_tp_methods
+Py_tp_new
+Py_tp_repr
+Py_tp_richcompare
+Py_tp_setattr
+Py_tp_setattro
+Py_tp_str
+Py_tp_traverse
+Py_uintptr_t
+_frame
+_node
+allocfunc
+binaryfunc
+descrgetfunc
+descrsetfunc
+destructor
+getattrfunc
+getattrofunc
+getiterfunc
+getter
+hashfunc
+initproc
+inquiry
+iternextfunc
+lenfunc
+newfunc
+objobjargproc
+objobjproc
+reprfunc
+richcmpfunc
+setattrfunc
+setattrofunc
+setter
+ssizeargfunc
+ssizeobjargproc
+ssizessizeargfunc
+ssizessizeobjargproc
+symtable
+ternaryfunc
+traverseproc
+unaryfunc
+visitproc

Makefile.pre.in

@@ -752,10 +752,7 @@ regen-importlib: Programs/_freeze_importlib
 
 
 regen-limited-abi: all
-	@$(MKDIR_P) $(srcdir)/Doc/data/
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py generate $(srcdir)/Doc/data/stable_abi.dat.new
-	$(UPDATE_FILE) $(srcdir)/Doc/data/stable_abi.dat \
-	$(srcdir)/Doc/data/stable_abi.dat.new
 
 
 ############################################################################
@@ -1983,7 +1980,7 @@ patchcheck: @DEF_MAKE_RULE@
 	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py
 
 check-limited-abi: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py check $(srcdir)/Doc/data/stable_abi.dat
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.txt
 
 # Dependencies
 

Misc/NEWS.d/next/C API/2021-04-09-18-19-07.bpo-43795.l0yobT.rst

@@ -0,0 +1,2 @@
+Stable ABI and limited API definitions are generated from a central manifest
+(:pep:`652`).

PC/python3dll.c

@@ -1,4 +1,7 @@
-/* Re-export stable Python API */
+
+/* Re-export stable Python ABI */
+
+/* Generated by Tools/scripts/stable_abi.py */
 
 #ifdef _M_IX86
 #define DECORATE "_"
@@ -41,6 +44,7 @@ EXPORT_FUNC(_PyTrash_thread_destroy_chain)
 EXPORT_FUNC(Py_AddPendingCall)
 EXPORT_FUNC(Py_AtExit)
 EXPORT_FUNC(Py_BuildValue)
+EXPORT_FUNC(Py_BytesMain)
 EXPORT_FUNC(Py_CompileString)
 EXPORT_FUNC(Py_DecodeLocale)
 EXPORT_FUNC(Py_DecRef)
@@ -70,7 +74,11 @@ EXPORT_FUNC(Py_GetVersion)
 EXPORT_FUNC(Py_IncRef)
 EXPORT_FUNC(Py_Initialize)
 EXPORT_FUNC(Py_InitializeEx)
+EXPORT_FUNC(Py_Is)
+EXPORT_FUNC(Py_IsFalse)
 EXPORT_FUNC(Py_IsInitialized)
+EXPORT_FUNC(Py_IsNone)
+EXPORT_FUNC(Py_IsTrue)
 EXPORT_FUNC(Py_LeaveRecursiveCall)
 EXPORT_FUNC(Py_Main)
 EXPORT_FUNC(Py_MakePendingCalls)
@@ -84,6 +92,7 @@ EXPORT_FUNC(Py_SetPythonHome)
 EXPORT_FUNC(Py_SetRecursionLimit)
 EXPORT_FUNC(Py_VaBuildValue)
 EXPORT_FUNC(Py_XNewRef)
+EXPORT_FUNC(PyAiter_Check)
 EXPORT_FUNC(PyArg_Parse)
 EXPORT_FUNC(PyArg_ParseTuple)
 EXPORT_FUNC(PyArg_ParseTupleAndKeywords)
@@ -230,14 +239,12 @@ EXPORT_FUNC(PyEval_EvalCodeEx)
 EXPORT_FUNC(PyEval_EvalFrame)
 EXPORT_FUNC(PyEval_EvalFrameEx)
 EXPORT_FUNC(PyEval_GetBuiltins)
-EXPORT_FUNC(PyEval_GetCallStats)
 EXPORT_FUNC(PyEval_GetFrame)
 EXPORT_FUNC(PyEval_GetFuncDesc)
 EXPORT_FUNC(PyEval_GetFuncName)
 EXPORT_FUNC(PyEval_GetGlobals)
 EXPORT_FUNC(PyEval_GetLocals)
 EXPORT_FUNC(PyEval_InitThreads)
-EXPORT_FUNC(PyEval_ReInitThreads)
 EXPORT_FUNC(PyEval_ReleaseLock)
 EXPORT_FUNC(PyEval_ReleaseThread)
 EXPORT_FUNC(PyEval_RestoreThread)
@@ -270,7 +277,6 @@ EXPORT_FUNC(PyGILState_Release)
 EXPORT_FUNC(PyImport_AddModule)
 EXPORT_FUNC(PyImport_AddModuleObject)
 EXPORT_FUNC(PyImport_AppendInittab)
-EXPORT_FUNC(PyImport_Cleanup)
 EXPORT_FUNC(PyImport_ExecCodeModule)
 EXPORT_FUNC(PyImport_ExecCodeModuleEx)
 EXPORT_FUNC(PyImport_ExecCodeModuleObject)
@@ -291,6 +297,8 @@ EXPORT_FUNC(PyImport_ReloadModule)
 EXPORT_FUNC(PyIndex_Check)
 EXPORT_FUNC(PyInterpreterState_Clear)
 EXPORT_FUNC(PyInterpreterState_Delete)
+EXPORT_FUNC(PyInterpreterState_Get)
+EXPORT_FUNC(PyInterpreterState_GetDict)
 EXPORT_FUNC(PyInterpreterState_GetID)
 EXPORT_FUNC(PyInterpreterState_New)
 EXPORT_FUNC(PyIter_Check)
@@ -339,10 +347,14 @@ EXPORT_FUNC(PyMapping_Length)
 EXPORT_FUNC(PyMapping_SetItemString)
 EXPORT_FUNC(PyMapping_Size)
 EXPORT_FUNC(PyMapping_Values)
+EXPORT_FUNC(PyMarshal_ReadObjectFromString)
+EXPORT_FUNC(PyMarshal_WriteObjectToString)
 EXPORT_FUNC(PyMem_Calloc)
 EXPORT_FUNC(PyMem_Free)
 EXPORT_FUNC(PyMem_Malloc)
 EXPORT_FUNC(PyMem_Realloc)
+EXPORT_FUNC(PyMember_GetOne)
+EXPORT_FUNC(PyMember_SetOne)
 EXPORT_FUNC(PyMemoryView_FromMemory)
 EXPORT_FUNC(PyMemoryView_FromObject)
 EXPORT_FUNC(PyMemoryView_GetContiguous)
@@ -421,12 +433,15 @@ EXPORT_FUNC(PyObject_Dir)
 EXPORT_FUNC(PyObject_Format)
 EXPORT_FUNC(PyObject_Free)
 EXPORT_FUNC(PyObject_GC_Del)
+EXPORT_FUNC(PyObject_GC_IsFinalized)
+EXPORT_FUNC(PyObject_GC_IsTracked)
 EXPORT_FUNC(PyObject_GC_Track)
 EXPORT_FUNC(PyObject_GC_UnTrack)
 EXPORT_FUNC(PyObject_GenericGetAttr)
 EXPORT_FUNC(PyObject_GenericGetDict)
 EXPORT_FUNC(PyObject_GenericSetAttr)
 EXPORT_FUNC(PyObject_GenericSetDict)
+EXPORT_FUNC(PyObject_GetAiter)
 EXPORT_FUNC(PyObject_GetAttr)
 EXPORT_FUNC(PyObject_GetAttrString)
 EXPORT_FUNC(PyObject_GetItem)
@@ -454,11 +469,6 @@ EXPORT_FUNC(PyObject_SetItem)
 EXPORT_FUNC(PyObject_Size)
 EXPORT_FUNC(PyObject_Str)
 EXPORT_FUNC(PyObject_Type)
-EXPORT_FUNC(PyODict_DelItem)
-EXPORT_FUNC(PyODict_New)
-EXPORT_FUNC(PyODict_SetItem)
-EXPORT_FUNC(PyOS_AfterFork)
-EXPORT_FUNC(PyOS_CheckStack)
 EXPORT_FUNC(PyOS_double_to_string)
 EXPORT_FUNC(PyOS_FSPath)
 EXPORT_FUNC(PyOS_getsig)
@@ -471,9 +481,6 @@ EXPORT_FUNC(PyOS_string_to_double)
 EXPORT_FUNC(PyOS_strtol)
 EXPORT_FUNC(PyOS_strtoul)
 EXPORT_FUNC(PyOS_vsnprintf)
-EXPORT_FUNC(PyParser_SimpleParseFileFlags)
-EXPORT_FUNC(PyParser_SimpleParseStringFlags)
-EXPORT_FUNC(PyParser_SimpleParseStringFlagsFilename)
 EXPORT_FUNC(PySeqIter_New)
 EXPORT_FUNC(PySequence_Check)
 EXPORT_FUNC(PySequence_Concat)
@@ -529,6 +536,25 @@ EXPORT_FUNC(PySys_SetObject)
 EXPORT_FUNC(PySys_SetPath)
 EXPORT_FUNC(PySys_WriteStderr)
 EXPORT_FUNC(PySys_WriteStdout)
+EXPORT_FUNC(PyThread_acquire_lock)
+EXPORT_FUNC(PyThread_acquire_lock_timed)
+EXPORT_FUNC(PyThread_allocate_lock)
+EXPORT_FUNC(PyThread_create_key)
+EXPORT_FUNC(PyThread_delete_key)
+EXPORT_FUNC(PyThread_delete_key_value)
+EXPORT_FUNC(PyThread_exit_thread)
+EXPORT_FUNC(PyThread_free_lock)
+EXPORT_FUNC(PyThread_get_key_value)
+EXPORT_FUNC(PyThread_get_stacksize)
+EXPORT_FUNC(PyThread_get_thread_ident)
+EXPORT_FUNC(PyThread_get_thread_native_id)
+EXPORT_FUNC(PyThread_GetInfo)
+EXPORT_FUNC(PyThread_init_thread)
+EXPORT_FUNC(PyThread_ReInitTLS)
+EXPORT_FUNC(PyThread_release_lock)
+EXPORT_FUNC(PyThread_set_key_value)
+EXPORT_FUNC(PyThread_set_stacksize)
+EXPORT_FUNC(PyThread_start_new_thread)
 EXPORT_FUNC(PyThread_tss_alloc)
 EXPORT_FUNC(PyThread_tss_create)
 EXPORT_FUNC(PyThread_tss_delete)
@@ -688,8 +714,6 @@ EXPORT_DATA(_Py_NoneStruct)
 EXPORT_DATA(_Py_NotImplementedStruct)
 EXPORT_DATA(_Py_SwappedOp)
 EXPORT_DATA(_Py_TrueStruct)
-EXPORT_DATA(_PyTrash_delete_later)
-EXPORT_DATA(_PyTrash_delete_nesting)
 EXPORT_DATA(_PyWeakref_CallableProxyType)
 EXPORT_DATA(_PyWeakref_ProxyType)
 EXPORT_DATA(_PyWeakref_RefType)
@@ -707,6 +731,7 @@ EXPORT_DATA(PyCallIter_Type)
 EXPORT_DATA(PyCapsule_Type)
 EXPORT_DATA(PyCFunction_Type)
 EXPORT_DATA(PyClassMethodDescr_Type)
+EXPORT_DATA(PyCodec_Unregister)
 EXPORT_DATA(PyComplex_Type)
 EXPORT_DATA(PyDict_Type)
 EXPORT_DATA(PyDictItems_Type)
@@ -715,6 +740,9 @@ EXPORT_DATA(PyDictIterKey_Type)
 EXPORT_DATA(PyDictIterValue_Type)
 EXPORT_DATA(PyDictKeys_Type)
 EXPORT_DATA(PyDictProxy_Type)
+EXPORT_DATA(PyDictRevIterItem_Type)
+EXPORT_DATA(PyDictRevIterKey_Type)
+EXPORT_DATA(PyDictRevIterValue_Type)
 EXPORT_DATA(PyDictValues_Type)
 EXPORT_DATA(PyEllipsis_Type)
 EXPORT_DATA(PyEnum_Type)
@@ -801,12 +829,6 @@ EXPORT_DATA(PyMemoryView_Type)
 EXPORT_DATA(PyMethodDescr_Type)
 EXPORT_DATA(PyModule_Type)
 EXPORT_DATA(PyModuleDef_Type)
-EXPORT_DATA(PyNullImporter_Type)
-EXPORT_DATA(PyODict_Type)
-EXPORT_DATA(PyODictItems_Type)
-EXPORT_DATA(PyODictIter_Type)
-EXPORT_DATA(PyODictKeys_Type)
-EXPORT_DATA(PyODictValues_Type)
 EXPORT_DATA(PyOS_InputHook)
 EXPORT_DATA(PyProperty_Type)
 EXPORT_DATA(PyRange_Type)
@@ -816,7 +838,6 @@ EXPORT_DATA(PySeqIter_Type)
 EXPORT_DATA(PySet_Type)
 EXPORT_DATA(PySetIter_Type)
 EXPORT_DATA(PySlice_Type)
-EXPORT_DATA(PySortWrapper_Type)
 EXPORT_DATA(PySuper_Type)
 EXPORT_DATA(PyTraceBack_Type)
 EXPORT_DATA(PyTuple_Type)

Tools/scripts/README

@@ -58,6 +58,7 @@ reindent-rst.py           Fix-up reStructuredText file whitespace
 rgrep.py                  Reverse grep through a file (useful for big logfiles)
 run_tests.py              Run the test suite with more sensible default options
 serve.py                  Small wsgiref-based web server, used in make serve in Doc
+stable_abi.py             Stable ABI checks and file generators.
 suff.py                   Sort a list of files by suffix
 texi2html.py              Convert GNU texinfo files into HTML
 untabify.py               Replace tabs with spaces in argument files

Tools/scripts/stable_abi.py

@@ -1,13 +1,28 @@
-#!/usr/bin/env python
+"""Check the stable ABI manifest or generate files from it
 
-import argparse
+By default, the tool only checks existing files/libraries.
-import glob
+Pass --generate to recreate auto-generated files instead.
-import os.path
+
-import pathlib
+For actions that take a FILENAME, the filename can be left out to use a default
-import re
+(relative to the manifest file, as they appear in the CPython codebase).
+"""
+
+from functools import partial
+from pathlib import Path
+import dataclasses
 import subprocess
-import sys
 import sysconfig
+import argparse
+import textwrap
+import difflib
+import shutil
+import sys
+import os
+import os.path
+import io
+import re
+
+MISSING = object()
 
 EXCLUDED_HEADERS = {
     "bytes_methods.h",
@@ -27,10 +42,303 @@ EXCLUDED_HEADERS = {
     "token.h",
     "ucnhash.h",
 }
-
 MACOS = (sys.platform == "darwin")
+UNIXY = MACOS or (sys.platform == "linux")  # XXX should this be "not Windows"?
 
-def get_exported_symbols(library, dynamic=False):
+
+# The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
+# following dataclasses.
+# Feel free to change its syntax (and the `parse_manifest` function)
+# to better serve that purpose (while keeping it human-readable).
+
+@dataclasses.dataclass
+class Manifest:
+    """Collection of `ABIItem`s forming the stable ABI/limited API."""
+
+    kind = 'manifest'
+    contents: dict = dataclasses.field(default_factory=dict)
+
+    def add(self, item):
+        if item.name in self.contents:
+            # We assume that stable ABI items do not share names,
+            # even if they're diferent kinds (e.g. function vs. macro).
+            raise ValueError(f'duplicate ABI item {item.name}')
+        self.contents[item.name] = item
+
+    @property
+    def feature_defines(self):
+        """Return all feature defines which affect what's available
+
+        These are e.g. HAVE_FORK and MS_WINDOWS.
+        """
+        return set(item.ifdef for item in self.contents.values()) - {None}
+
+    def select(self, kinds, *, include_abi_only=True, ifdef=None):
+        """Yield selected items of the manifest
+
+        kinds: set of requested kinds, e.g. {'function', 'macro'}
+        include_abi_only: if True (default), include all items of the
+            stable ABI.
+            If False, include only items from the limited API
+            (i.e. items people should use today)
+        ifdef: set of feature defines (e.g. {'HAVE_FORK', 'MS_WINDOWS'}).
+            If None (default), items are not filtered by this. (This is
+            different from the empty set, which filters out all such
+            conditional items.)
+        """
+        for name, item in sorted(self.contents.items()):
+            if item.kind not in kinds:
+                continue
+            if item.abi_only and not include_abi_only:
+                continue
+            if (ifdef is not None
+                    and item.ifdef is not None
+                    and item.ifdef not in ifdef):
+                continue
+            yield item
+
+    def dump(self):
+        """Yield lines to recreate the manifest file (sans comments/newlines)"""
+        # Recursive in preparation for struct member & function argument nodes
+        for item in self.contents.values():
+            yield from item.dump(indent=0)
+
+@dataclasses.dataclass
+class ABIItem:
+    """Information on one item (function, macro, struct, etc.)"""
+
+    kind: str
+    name: str
+    added: str = None
+    contents: list = dataclasses.field(default_factory=list)
+    abi_only: bool = False
+    ifdef: str = None
+
+    KINDS = frozenset({
+        'struct', 'function', 'macro', 'data', 'const', 'typedef',
+    })
+
+    def dump(self, indent=0):
+        yield f"{'    ' * indent}{self.kind} {self.name}"
+        if self.added:
+            yield f"{'    ' * (indent+1)}added {self.added}"
+        if self.ifdef:
+            yield f"{'    ' * (indent+1)}ifdef {self.ifdef}"
+        if self.abi_only:
+            yield f"{'    ' * (indent+1)}abi_only"
+
+def parse_manifest(file):
+    """Parse the given file (iterable of lines) to a Manifest"""
+
+    LINE_RE = re.compile('(?P<indent>[ ]*)(?P<kind>[^ ]+)[ ]*(?P<content>.*)')
+    manifest = Manifest()
+
+    # parents of currently processed line, each with its indentation level
+    levels = [(manifest, -1)]
+
+    def raise_error(msg):
+        raise SyntaxError(f'line {lineno}: {msg}')
+
+    for lineno, line in enumerate(file, start=1):
+        line, sep, comment = line.partition('#')
+        line = line.rstrip()
+        if not line:
+            continue
+        match = LINE_RE.fullmatch(line)
+        if not match:
+            raise_error(f'invalid syntax: {line}')
+        level = len(match['indent'])
+        kind = match['kind']
+        content = match['content']
+        while level <= levels[-1][1]:
+            levels.pop()
+        parent = levels[-1][0]
+        entry = None
+        if kind in ABIItem.KINDS:
+            if parent.kind not in {'manifest'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            entry = ABIItem(kind, content)
+            parent.add(entry)
+        elif kind in {'added', 'ifdef'}:
+            if parent.kind not in ABIItem.KINDS:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            setattr(parent, kind, content)
+        elif kind in {'abi_only'}:
+            if parent.kind not in {'function', 'data'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            parent.abi_only = True
+        else:
+            raise_error(f"unknown kind {kind!r}")
+        levels.append((entry, level))
+    return manifest
+
+# The tool can run individual "actions".
+# Most actions are "generators", which generate a single file from the
+# manifest. (Checking works by generating a temp file & comparing.)
+# Other actions, like "--unixy-check", don't work on a single file.
+
+generators = []
+def generator(var_name, default_path):
+    """Decorates a file generator: function that writes to a file"""
+    def _decorator(func):
+        func.var_name = var_name
+        func.arg_name = '--' + var_name.replace('_', '-')
+        func.default_path = default_path
+        generators.append(func)
+        return func
+    return _decorator
+
+
+@generator("python3dll", 'PC/python3dll.c')
+def gen_python3dll(manifest, args, outfile):
+    """Generate/check the source for the Windows stable ABI library"""
+    write = partial(print, file=outfile)
+    write(textwrap.dedent(r"""
+        /* Re-export stable Python ABI */
+
+        /* Generated by Tools/scripts/stable_abi.py */
+
+        #ifdef _M_IX86
+        #define DECORATE "_"
+        #else
+        #define DECORATE
+        #endif
+
+        #define EXPORT_FUNC(name) \
+            __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name))
+        #define EXPORT_DATA(name) \
+            __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name ",DATA"))
+    """))
+
+    def sort_key(item):
+        return item.name.lower()
+
+    for item in sorted(
+            manifest.select(
+                {'function'}, include_abi_only=True, ifdef={'MS_WINDOWS'}),
+            key=sort_key):
+        write(f'EXPORT_FUNC({item.name})')
+
+    write()
+
+    for item in sorted(
+            manifest.select(
+                {'data'}, include_abi_only=True, ifdef={'MS_WINDOWS'}),
+            key=sort_key):
+        write(f'EXPORT_DATA({item.name})')
+
+
+@generator("doc_list", 'Doc/data/stable_abi.dat')
+def gen_doc_annotations(manifest, args, outfile):
+    """Generate/check the stable ABI list for documentation annotations"""
+    write = partial(print, file=outfile)
+    write("# Generated by Tools/scripts/stable_abi.py")
+    write()
+    for item in manifest.select(ABIItem.KINDS, include_abi_only=False):
+        write(item.name)
+
+
+def generate_or_check(manifest, args, path, func):
+    """Generate/check a file with a single generator
+
+    Return True if successful; False if a comparison failed.
+    """
+
+    outfile = io.StringIO()
+    func(manifest, args, outfile)
+    generated = outfile.getvalue()
+    existing = path.read_text()
+
+    if generated != existing:
+        if args.generate:
+            path.write_text(generated)
+        else:
+            print(f'File {path} differs from expected!')
+            diff = difflib.unified_diff(
+                generated.splitlines(), existing.splitlines(),
+                str(path), '<expected>',
+                lineterm='',
+            )
+            for line in diff:
+                print(line)
+            return False
+    return True
+
+
+def do_unixy_check(manifest, args):
+    """Check headers & library using "Unixy" tools (GCC/clang, binutils)"""
+    okay = True
+
+    # Get all macros first: we'll need feature macros like HAVE_FORK and
+    # MS_WINDOWS for everything else
+    present_macros = gcc_get_limited_api_macros(['Include/Python.h'])
+    feature_defines = manifest.feature_defines & present_macros
+
+    # Check that we have all neded macros
+    expected_macros = set(
+        item.name for item in manifest.select({'macro'})
+    )
+    missing_macros = expected_macros - present_macros
+    okay &= _report_unexpected_items(
+        missing_macros,
+        'Some macros from are not defined from "Include/Python.h"'
+        + 'with Py_LIMITED_API:')
+
+    expected_symbols = set(item.name for item in manifest.select(
+        {'function', 'data'}, include_abi_only=True, ifdef=feature_defines,
+    ))
+
+    # Check the static library (*.a)
+    LIBRARY = sysconfig.get_config_var("LIBRARY")
+    if not LIBRARY:
+        raise Exception("failed to get LIBRARY variable from sysconfig")
+    if os.path.exists(LIBRARY):
+        okay &= binutils_check_library(
+            manifest, LIBRARY, expected_symbols, dynamic=False)
+
+    # Check the dynamic library (*.so)
+    LDLIBRARY = sysconfig.get_config_var("LDLIBRARY")
+    if not LDLIBRARY:
+        raise Exception("failed to get LDLIBRARY variable from sysconfig")
+    okay &= binutils_check_library(
+            manifest, LDLIBRARY, expected_symbols, dynamic=False)
+
+    # Check definitions in the header files
+    expected_defs = set(item.name for item in manifest.select(
+        {'function', 'data'}, include_abi_only=False, ifdef=feature_defines,
+    ))
+    found_defs = gcc_get_limited_api_definitions(['Include/Python.h'])
+    missing_defs = expected_defs - found_defs
+    okay &= _report_unexpected_items(
+        missing_defs,
+        'Some expected declarations were not declared in '
+        + '"Include/Python.h" with Py_LIMITED_API:')
+
+    # Some Limited API macros are defined in terms of private symbols.
+    # These are not part of Limited API (even though they're defined with
+    # Py_LIMITED_API). They must be part of the Stable ABI, though.
+    private_symbols = {n for n in expected_symbols if n.startswith('_')}
+    extra_defs = found_defs - expected_defs - private_symbols
+    okay &= _report_unexpected_items(
+        extra_defs,
+        'Some extra declarations were found in "Include/Python.h" '
+        + 'with Py_LIMITED_API:')
+
+    return okay
+
+
+def _report_unexpected_items(items, msg):
+    """If there are any `items`, report them using "msg" and return false"""
+    if items:
+        print(msg, file=sys.stderr)
+        for item in sorted(items):
+            print(' -', item, file=sys.stderr)
+        return False
+    return True
+
+
+def binutils_get_exported_symbols(library, dynamic=False):
+    """Retrieve exported symbols using the nm(1) tool from binutils"""
     # Only look at dynamic symbols
     args = ["nm", "--no-sort"]
     if dynamic:
@@ -61,86 +369,89 @@ def get_exported_symbols(library, dynamic=False):
             yield symbol
 
 
-def check_library(stable_abi_file, library, abi_funcs, dynamic=False):
+def binutils_check_library(manifest, library, expected_symbols, dynamic):
-    available_symbols = set(get_exported_symbols(library, dynamic))
+    """Check that library exports all expected_symbols"""
-    missing_symbols = abi_funcs - available_symbols
+    available_symbols = set(binutils_get_exported_symbols(library, dynamic))
+    missing_symbols = expected_symbols - available_symbols
     if missing_symbols:
-        raise Exception(
+        print(textwrap.dedent(f"""\
-            f"""\
+            Some symbols from the limited API are missing from {library}:
-Some symbols from the limited API are missing: {', '.join(missing_symbols)}
+                {', '.join(missing_symbols)}
 
-This error means that there are some missing symbols among the ones exported
+            This error means that there are some missing symbols among the
-in the Python library ("libpythonx.x.a" or "libpythonx.x.so"). This normally
+            ones exported in the library.
-means that some symbol, function implementation or a prototype, belonging to
+            This normally means that some symbol, function implementation or
-a symbol in the limited API has been deleted or is missing.
+            a prototype belonging to a symbol in the limited API has been
-
+            deleted or is missing.
-Check if this was a mistake and if not, update the file containing the limited
+        """), file=sys.stderr)
-API symbols. This file is located at:
+        return False
-
+    return True
-{stable_abi_file}
-
-You can read more about the limited API and its contracts at:
-
-https://docs.python.org/3/c-api/stable.html
-
-And in PEP 384:
-
-https://www.python.org/dev/peps/pep-0384/
-"""
-        )
 
 
-def generate_limited_api_symbols(args):
+def gcc_get_limited_api_macros(headers):
-    library = sysconfig.get_config_var("LIBRARY")
+    """Get all limited API macros from headers.
-    ldlibrary = sysconfig.get_config_var("LDLIBRARY")
-    if ldlibrary != library:
-        raise Exception("Limited ABI symbols can only be generated from a static build")
-    available_symbols = {
-        symbol for symbol in get_exported_symbols(library) if symbol.startswith("Py")
-    }
 
-    headers = [
+    Runs the preprocesor over all the header files in "Include" setting
-        file
+    "-DPy_LIMITED_API" to the correct value for the running version of the
-        for file in pathlib.Path("Include").glob("*.h")
+    interpreter and extracting all macro definitions (via adding -dM to the
-        if file.name not in EXCLUDED_HEADERS
+    compiler arguments).
-    ]
+
-    stable_data, stable_exported_data, stable_functions = get_limited_api_definitions(
+    Requires Python built with a GCC-compatible compiler. (clang might work)
-        headers
+    """
+
+    api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16
+
+    preprocesor_output_with_macros = subprocess.check_output(
+        sysconfig.get_config_var("CC").split()
+        + [
+            # Prevent the expansion of the exported macros so we can
+            # capture them later
+            "-DSIZEOF_WCHAR_T=4",  # The actual value is not important
+            f"-DPy_LIMITED_API={api_hexversion}",
+            "-I.",
+            "-I./Include",
+            "-dM",
+            "-E",
+        ]
+        + [str(file) for file in headers],
+        text=True,
     )
 
-    stable_symbols = {
+    return {
-        symbol
+        target
-        for symbol in (stable_functions | stable_exported_data | stable_data)
+        for target in re.findall(
-        if symbol.startswith("Py") and symbol in available_symbols
+            r"#define (\w+)", preprocesor_output_with_macros
-    }
-    with open(args.output_file, "w") as output_file:
-        output_file.write(f"# File generated by 'make regen-limited-abi'\n")
-        output_file.write(
-            f"# This is NOT an authoritative list of stable ABI symbols\n"
         )
-        for symbol in sorted(stable_symbols):
+    }
-            output_file.write(f"{symbol}\n")
 
 
-def get_limited_api_definitions(headers):
+def gcc_get_limited_api_definitions(headers):
-    """Run the preprocesor over all the header files in "Include" setting
+    """Get all limited API definitions from headers.
-    "-DPy_LIMITED_API" to the correct value for the running version of the interpreter.
 
-    The limited API symbols will be extracted from the output of this command as it includes
+    Run the preprocesor over all the header files in "Include" setting
-    the prototypes and definitions of all the exported symbols that are in the limited api.
+    "-DPy_LIMITED_API" to the correct value for the running version of the
+    interpreter.
+
+    The limited API symbols will be extracted from the output of this command
+    as it includes the prototypes and definitions of all the exported symbols
+    that are in the limited api.
 
     This function does *NOT* extract the macros defined on the limited API
+
+    Requires Python built with a GCC-compatible compiler. (clang might work)
     """
+    api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16
     preprocesor_output = subprocess.check_output(
         sysconfig.get_config_var("CC").split()
         + [
-            # Prevent the expansion of the exported macros so we can capture them later
+            # Prevent the expansion of the exported macros so we can capture
+            # them later
             "-DPyAPI_FUNC=__PyAPI_FUNC",
             "-DPyAPI_DATA=__PyAPI_DATA",
             "-DEXPORT_DATA=__EXPORT_DATA",
             "-D_Py_NO_RETURN=",
             "-DSIZEOF_WCHAR_T=4",  # The actual value is not important
-            f"-DPy_LIMITED_API={sys.version_info.major << 24 | sys.version_info.minor << 16}",
+            f"-DPy_LIMITED_API={api_hexversion}",
             "-I.",
             "-I./Include",
             "-E",
@@ -156,64 +467,126 @@ def get_limited_api_definitions(headers):
         re.findall(r"__EXPORT_DATA\((.*?)\)", preprocesor_output)
     )
     stable_data = set(
-        re.findall(r"__PyAPI_DATA\(.*?\)\s*\(?(.*?)\)?\s*;", preprocesor_output)
+        re.findall(r"__PyAPI_DATA\(.*?\)[\s\*\(]*([^);]*)\)?.*;", preprocesor_output)
     )
-    return stable_data, stable_exported_data, stable_functions
+    return stable_data | stable_exported_data | stable_functions
-
-
-def check_symbols(parser_args):
-    with open(parser_args.stable_abi_file, "r") as filename:
-        abi_funcs = {
-            symbol
-            for symbol in filename.read().splitlines()
-            if symbol and not symbol.startswith("#")
-        }
-
-    try:
-        # static library
-        LIBRARY = sysconfig.get_config_var("LIBRARY")
-        if not LIBRARY:
-            raise Exception("failed to get LIBRARY variable from sysconfig")
-        if os.path.exists(LIBRARY):
-            check_library(parser_args.stable_abi_file, LIBRARY, abi_funcs)
-
-        # dynamic library
-        LDLIBRARY = sysconfig.get_config_var("LDLIBRARY")
-        if not LDLIBRARY:
-            raise Exception("failed to get LDLIBRARY variable from sysconfig")
-        if LDLIBRARY != LIBRARY:
-            check_library(
-                parser_args.stable_abi_file, LDLIBRARY, abi_funcs, dynamic=True
-            )
-    except Exception as e:
-        print(e, file=sys.stderr)
-        sys.exit(1)
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Process some integers.")
+    parser = argparse.ArgumentParser(
-    subparsers = parser.add_subparsers()
+        description=__doc__,
-    check_parser = subparsers.add_parser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
-        "check", help="Check the exported symbols against a given ABI file"
     )
-    check_parser.add_argument(
+    parser.add_argument(
-        "stable_abi_file", type=str, help="File with the stable abi functions"
+        "file", type=Path, metavar='FILE',
+        help="file with the stable abi manifest",
     )
-    check_parser.set_defaults(func=check_symbols)
+    parser.add_argument(
-    generate_parser = subparsers.add_parser(
+        "--generate", action='store_true',
-        "generate",
+        help="generate file(s), rather than just checking them",
-        help="Generate symbols from the header files and the exported symbols",
     )
-    generate_parser.add_argument(
+    parser.add_argument(
-        "output_file", type=str, help="File to dump the symbols to"
+        "--generate-all", action='store_true',
+        help="as --generate, but generate all file(s) using default filenames."
+            + " (unlike --all, does not run any extra checks)",
+    )
+    parser.add_argument(
+        "-a", "--all", action='store_true',
+        help="run all available checks using default filenames",
+    )
+    parser.add_argument(
+        "-l", "--list", action='store_true',
+        help="list available generators and their default filenames; then exit",
+    )
+    parser.add_argument(
+        "--dump", action='store_true',
+        help="dump the manifest contents (used for debugging the parser)",
     )
-    generate_parser.set_defaults(func=generate_limited_api_symbols)
-    args = parser.parse_args()
-    if "func" not in args:
-        parser.error("Either 'check' or 'generate' must be used")
-        sys.exit(1)
 
-    args.func(args)
+    actions_group = parser.add_argument_group('actions')
+    for gen in generators:
+        actions_group.add_argument(
+            gen.arg_name, dest=gen.var_name,
+            type=str, nargs="?", default=MISSING,
+            metavar='FILENAME',
+            help=gen.__doc__,
+        )
+    actions_group.add_argument(
+        '--unixy-check', action='store_true',
+        help=do_unixy_check.__doc__,
+    )
+    args = parser.parse_args()
+
+    base_path = args.file.parent.parent
+
+    if args.list:
+        for gen in generators:
+            print(f'{gen.arg_name}: {base_path / gen.default_path}')
+        sys.exit(0)
+
+    run_all_generators = args.generate_all
+
+    if args.generate_all:
+        args.generate = True
+
+    if args.all:
+        run_all_generators = True
+        args.unixy_check = True
+
+    with args.file.open() as file:
+        manifest = parse_manifest(file)
+
+    # Remember results of all actions (as booleans).
+    # At the end we'll check that at least one action was run,
+    # and also fail if any are false.
+    results = {}
+
+    if args.dump:
+        for line in manifest.dump():
+            print(line)
+        results['dump'] = True
+
+    for gen in generators:
+        filename = getattr(args, gen.var_name)
+        if filename is None or (run_all_generators and filename is MISSING):
+            filename = base_path / gen.default_path
+        elif filename is MISSING:
+            continue
+
+        results[gen.var_name] = generate_or_check(manifest, args, filename, gen)
+
+    if args.unixy_check:
+        results['unixy_check'] = do_unixy_check(manifest, args)
+
+    if not results:
+        if args.generate:
+            parser.error('No file specified. Use --help for usage.')
+        parser.error('No check specified. Use --help for usage.')
+
+    failed_results = [name for name, result in results.items() if not result]
+
+    if failed_results:
+        raise Exception(f"""
+        These checks related to the stable ABI did not succeed:
+            {', '.join(failed_results)}
+
+        If you see diffs in the output, files derived from the stable
+        ABI manifest the were not regenerated.
+        Run `make regen-limited-abi` to fix this.
+
+        Otherwise, see the error(s) above.
+
+        The stable ABI manifest is at: {args.file}
+        Note that there is a process to follow when modifying it.
+
+        You can read more about the limited API and its contracts at:
+
+        https://docs.python.org/3/c-api/stable.html
+
+        And in PEP 384:
+
+        https://www.python.org/dev/peps/pep-0384/
+        """)
 
 
 if __name__ == "__main__":