gh-91324: Convert the stable ABI manifest to TOML (GH-92026)

Lib/test/test_stable_abi_ctypes.py

@@ -20,7 +20,8 @@ class TestStableABIAvailability(unittest.TestCase):
                 ctypes_test.pythonapi[symbol_name]
 
     def test_feature_macros(self):
-        self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
+        self.assertEqual(
+            set(get_feature_macros()), EXPECTED_FEATURE_MACROS)
 
     # The feature macros for Windows are used in creating the DLL
     # definition, so they must be known on all platforms.
@@ -28,7 +29,7 @@ class TestStableABIAvailability(unittest.TestCase):
     # the reality.
     @unittest.skipIf(sys.platform != "win32", "Windows specific test")
     def test_windows_feature_macros(self):
-        for name, value in WINDOWS_IFDEFS.items():
+        for name, value in WINDOWS_FEATURE_MACROS.items():
             if value != 'maybe':
                 with self.subTest(name):
                     self.assertEqual(feature_macros[name], value)
@@ -909,5 +910,13 @@ if feature_macros['Py_REF_DEBUG']:
         '_Py_RefTotal',
     )
 
-EXPECTED_IFDEFS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', 'USE_STACKCHECK'])
+EXPECTED_FEATURE_MACROS = set(['HAVE_FORK',
-WINDOWS_IFDEFS = {'MS_WINDOWS': True, 'HAVE_FORK': False, 'USE_STACKCHECK': 'maybe', 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe'}
+ 'MS_WINDOWS',
+ 'PY_HAVE_THREAD_NATIVE_ID',
+ 'Py_REF_DEBUG',
+ 'USE_STACKCHECK'])
+WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False,
+ 'MS_WINDOWS': True,
+ 'PY_HAVE_THREAD_NATIVE_ID': True,
+ 'Py_REF_DEBUG': 'maybe',
+ 'USE_STACKCHECK': 'maybe'}

Makefile.pre.in

@@ -1199,7 +1199,7 @@ regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
 # ABI
 
 regen-limited-abi: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml
 
 ############################################################################
 # Regenerate all generated files
@@ -2476,7 +2476,7 @@ patchcheck: @DEF_MAKE_RULE@
 	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py
 
 check-limited-abi: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.txt
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml
 
 .PHONY: update-config
 update-config:

Tools/scripts/stable_abi.py

@@ -14,8 +14,10 @@ import subprocess
 import sysconfig
 import argparse
 import textwrap
+import tomllib
 import difflib
 import shutil
+import pprint
 import sys
 import os
 import os.path
@@ -46,17 +48,15 @@ MACOS = (sys.platform == "darwin")
 UNIXY = MACOS or (sys.platform == "linux")  # XXX should this be "not Windows"?
 
 
-# The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
+# The stable ABI manifest (Misc/stable_abi.toml) exists only to fill the
 # following dataclasses.
 # Feel free to change its syntax (and the `parse_manifest` function)
 # to better serve that purpose (while keeping it human-readable).
 
-@dataclasses.dataclass
 class Manifest:
     """Collection of `ABIItem`s forming the stable ABI/limited API."""
-
+    def __init__(self):
-    kind = 'manifest'
+        self.contents = dict()
-    contents: dict = dataclasses.field(default_factory=dict)
 
     def add(self, item):
         if item.name in self.contents:
@@ -65,14 +65,6 @@ class Manifest:
             raise ValueError(f'duplicate ABI item {item.name}')
         self.contents[item.name] = item
 
-    @property
-    def feature_defines(self):
-        """Return all feature defines which affect what's available
-
-        These are e.g. HAVE_FORK and MS_WINDOWS.
-        """
-        return set(item.ifdef for item in self.contents.values()) - {None}
-
     def select(self, kinds, *, include_abi_only=True, ifdef=None):
         """Yield selected items of the manifest
 
@@ -81,7 +73,7 @@ class Manifest:
             stable ABI.
             If False, include only items from the limited API
             (i.e. items people should use today)
-        ifdef: set of feature defines (e.g. {'HAVE_FORK', 'MS_WINDOWS'}).
+        ifdef: set of feature macros (e.g. {'HAVE_FORK', 'MS_WINDOWS'}).
             If None (default), items are not filtered by this. (This is
             different from the empty set, which filters out all such
             conditional items.)
@@ -99,109 +91,74 @@ class Manifest:
 
     def dump(self):
         """Yield lines to recreate the manifest file (sans comments/newlines)"""
-        # Recursive in preparation for struct member & function argument nodes
         for item in self.contents.values():
-            yield from item.dump(indent=0)
+            fields = dataclasses.fields(item)
+            yield f"[{item.kind}.{item.name}]"
+            for field in fields:
+                if field.name in {'name', 'value', 'kind'}:
+                    continue
+                value = getattr(item, field.name)
+                if value == field.default:
+                    pass
+                elif value is True:
+                    yield f"    {field.name} = true"
+                elif value:
+                    yield f"    {field.name} = {value!r}"
 
+
+itemclasses = {}
+def itemclass(kind):
+    """Register the decorated class in `itemclasses`"""
+    def decorator(cls):
+        itemclasses[kind] = cls
+        return cls
+    return decorator
+
+@itemclass('function')
+@itemclass('macro')
+@itemclass('data')
+@itemclass('const')
+@itemclass('typedef')
 @dataclasses.dataclass
 class ABIItem:
     """Information on one item (function, macro, struct, etc.)"""
 
-    kind: str
     name: str
+    kind: str
     added: str = None
-    contents: list = dataclasses.field(default_factory=list)
     abi_only: bool = False
     ifdef: str = None
-    struct_abi_kind: str = None
+
-    members: list = None
+@itemclass('feature_macro')
-    doc: str = None
+@dataclasses.dataclass(kw_only=True)
+class FeatureMacro(ABIItem):
+    name: str
+    doc: str
     windows: bool = False
+    abi_only: bool = True
 
-    KINDS = frozenset({
+@itemclass('struct')
-        'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef',
+@dataclasses.dataclass(kw_only=True)
-    })
+class Struct(ABIItem):
+    struct_abi_kind: str
+    members: list = None
 
-    def dump(self, indent=0):
-        yield f"{'    ' * indent}{self.kind} {self.name}"
-        if self.added:
-            yield f"{'    ' * (indent+1)}added {self.added}"
-        if self.ifdef:
-            yield f"{'    ' * (indent+1)}ifdef {self.ifdef}"
-        if self.abi_only:
-            yield f"{'    ' * (indent+1)}abi_only"
 
 def parse_manifest(file):
     """Parse the given file (iterable of lines) to a Manifest"""
 
-    LINE_RE = re.compile('(?P<indent>[ ]*)(?P<kind>[^ ]+)[ ]*(?P<content>.*)')
     manifest = Manifest()
 
-    # parents of currently processed line, each with its indentation level
+    data = tomllib.load(file)
-    levels = [(manifest, -1)]
 
-    def raise_error(msg):
+    for kind, itemclass in itemclasses.items():
-        raise SyntaxError(f'line {lineno}: {msg}')
+        for name, item_data in data[kind].items():
-
+            try:
-    for lineno, line in enumerate(file, start=1):
+                item = itemclass(name=name, kind=kind, **item_data)
-        line, sep, comment = line.partition('#')
+                manifest.add(item)
-        line = line.rstrip()
+            except BaseException as exc:
-        if not line:
+                exc.add_note(f'in {kind} {name}')
-            continue
+                raise
-        match = LINE_RE.fullmatch(line)
-        if not match:
-            raise_error(f'invalid syntax: {line}')
-        level = len(match['indent'])
-        kind = match['kind']
-        content = match['content']
-        while level <= levels[-1][1]:
-            levels.pop()
-        parent = levels[-1][0]
-        entry = None
-        if parent.kind == 'manifest':
-            if kind not in kind in ABIItem.KINDS:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            entry = ABIItem(kind, content)
-            parent.add(entry)
-        elif kind in {'added', 'ifdef'}:
-            if parent.kind not in ABIItem.KINDS:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            setattr(parent, kind, content)
-        elif kind in {'abi_only'}:
-            if parent.kind not in {'function', 'data'}:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            parent.abi_only = True
-        elif kind in {'members', 'full-abi', 'opaque'}:
-            if parent.kind not in {'struct'}:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            if prev := getattr(parent, 'struct_abi_kind', None):
-                raise_error(
-                    f'{parent.name} already has {prev}, cannot add {kind}')
-            parent.struct_abi_kind = kind
-            if kind == 'members':
-                parent.members = content.split()
-        elif kind in {'doc'}:
-            if parent.kind not in {'ifdef'}:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            parent.doc = content
-        elif kind in {'windows'}:
-            if parent.kind not in {'ifdef'}:
-                raise_error(f'{kind} cannot go in {parent.kind}')
-            if not content:
-                parent.windows = True
-            elif content == 'maybe':
-                parent.windows = content
-            else:
-                raise_error(f'Unexpected: {content}')
-        else:
-            raise_error(f"unknown kind {kind!r}")
-            # When adding more, update the comment in stable_abi.txt.
-        levels.append((entry, level))
-
-    ifdef_names = {i.name for i in manifest.select({'ifdef'})}
-    for item in manifest.contents.values():
-        if item.ifdef and item.ifdef not in ifdef_names:
-            raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}')
 
     return manifest
 
@@ -246,12 +203,14 @@ def gen_python3dll(manifest, args, outfile):
     def sort_key(item):
         return item.name.lower()
 
-    windows_ifdefs = {
+    windows_feature_macros = {
-        item.name for item in manifest.select({'ifdef'}) if item.windows
+        item.name for item in manifest.select({'feature_macro'}) if item.windows
     }
     for item in sorted(
             manifest.select(
-                {'function'}, include_abi_only=True, ifdef=windows_ifdefs),
+                {'function'},
+                include_abi_only=True,
+                ifdef=windows_feature_macros),
             key=sort_key):
         write(f'EXPORT_FUNC({item.name})')
 
@@ -259,7 +218,9 @@ def gen_python3dll(manifest, args, outfile):
 
     for item in sorted(
             manifest.select(
-                {'data'}, include_abi_only=True, ifdef=windows_ifdefs),
+                {'data'},
+                include_abi_only=True,
+                ifdef=windows_feature_macros),
             key=sort_key):
         write(f'EXPORT_DATA({item.name})')
 
@@ -285,17 +246,20 @@ def gen_doc_annotations(manifest, args, outfile):
             ifdef_note = manifest.contents[item.ifdef].doc
         else:
             ifdef_note = None
-        writer.writerow({
+        row = {
             'role': REST_ROLES[item.kind],
             'name': item.name,
             'added': item.added,
-            'ifdef_note': ifdef_note,
+            'ifdef_note': ifdef_note}
-            'struct_abi_kind': item.struct_abi_kind})
+        rows = [row]
-        for member_name in item.members or ():
+        if item.kind == 'struct':
-            writer.writerow({
+            row['struct_abi_kind'] = item.struct_abi_kind
-                'role': 'member',
+            for member_name in item.members or ():
-                'name': f'{item.name}.{member_name}',
+                rows.append({
-                'added': item.added})
+                    'role': 'member',
+                    'name': f'{item.name}.{member_name}',
+                    'added': item.added})
+        writer.writerows(rows)
 
 @generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py')
 def gen_ctypes_test(manifest, args, outfile):
@@ -323,7 +287,8 @@ def gen_ctypes_test(manifest, args, outfile):
                         ctypes_test.pythonapi[symbol_name]
 
             def test_feature_macros(self):
-                self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
+                self.assertEqual(
+                    set(get_feature_macros()), EXPECTED_FEATURE_MACROS)
 
             # The feature macros for Windows are used in creating the DLL
             # definition, so they must be known on all platforms.
@@ -331,7 +296,7 @@ def gen_ctypes_test(manifest, args, outfile):
             # the reality.
             @unittest.skipIf(sys.platform != "win32", "Windows specific test")
             def test_windows_feature_macros(self):
-                for name, value in WINDOWS_IFDEFS.items():
+                for name, value in WINDOWS_FEATURE_MACROS.items():
                     if value != 'maybe':
                         with self.subTest(name):
                             self.assertEqual(feature_macros[name], value)
@@ -342,7 +307,7 @@ def gen_ctypes_test(manifest, args, outfile):
         {'function', 'data'},
         include_abi_only=True,
     )
-    ifdef_items = {}
+    optional_items = {}
     for item in items:
         if item.name in (
                 # Some symbols aren't exported on all platforms.
@@ -351,23 +316,23 @@ def gen_ctypes_test(manifest, args, outfile):
             ):
             continue
         if item.ifdef:
-            ifdef_items.setdefault(item.ifdef, []).append(item.name)
+            optional_items.setdefault(item.ifdef, []).append(item.name)
         else:
             write(f'    "{item.name}",')
     write(")")
-    for ifdef, names in ifdef_items.items():
+    for ifdef, names in optional_items.items():
         write(f"if feature_macros[{ifdef!r}]:")
         write(f"    SYMBOL_NAMES += (")
         for name in names:
             write(f"        {name!r},")
         write("    )")
     write("")
-    write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})")
+    feature_macros = list(manifest.select({'feature_macro'}))
+    feature_names = sorted(m.name for m in feature_macros)
+    write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})")
 
-    windows_ifdef_values = {
+    windows_feature_macros = {m.name: m.windows for m in feature_macros}
-        name: manifest.contents[name].windows for name in ifdef_items
+    write(f"WINDOWS_FEATURE_MACROS = {pprint.pformat(windows_feature_macros)}")
-    }
-    write(f"WINDOWS_IFDEFS = {windows_ifdef_values}")
 
 
 @generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc')
@@ -378,7 +343,7 @@ def gen_testcapi_feature_macros(manifest, args, outfile):
     write()
     write('// Add an entry in dict `result` for each Stable ABI feature macro.')
     write()
-    for macro in manifest.select({'ifdef'}):
+    for macro in manifest.select({'feature_macro'}):
         name = macro.name
         write(f'#ifdef {name}')
         write(f'    res = PyDict_SetItemString(result, "{name}", Py_True);')
@@ -425,7 +390,8 @@ def do_unixy_check(manifest, args):
     # Get all macros first: we'll need feature macros like HAVE_FORK and
     # MS_WINDOWS for everything else
     present_macros = gcc_get_limited_api_macros(['Include/Python.h'])
-    feature_defines = manifest.feature_defines & present_macros
+    feature_macros = set(m.name for m in manifest.select({'feature_macro'}))
+    feature_macros &= present_macros
 
     # Check that we have all needed macros
     expected_macros = set(
@@ -438,7 +404,7 @@ def do_unixy_check(manifest, args):
         + 'with Py_LIMITED_API:')
 
     expected_symbols = set(item.name for item in manifest.select(
-        {'function', 'data'}, include_abi_only=True, ifdef=feature_defines,
+        {'function', 'data'}, include_abi_only=True, ifdef=feature_macros,
     ))
 
     # Check the static library (*.a)
@@ -458,7 +424,7 @@ def do_unixy_check(manifest, args):
 
     # Check definitions in the header files
     expected_defs = set(item.name for item in manifest.select(
-        {'function', 'data'}, include_abi_only=False, ifdef=feature_defines,
+        {'function', 'data'}, include_abi_only=False, ifdef=feature_macros,
     ))
     found_defs = gcc_get_limited_api_definitions(['Include/Python.h'])
     missing_defs = expected_defs - found_defs
@@ -635,6 +601,28 @@ def check_private_names(manifest):
                 f'`{name}` is private (underscore-prefixed) and should be '
                 + 'removed from the stable ABI list or or marked `abi_only`')
 
+def check_dump(manifest, filename):
+    """Check that manifest.dump() corresponds to the data.
+
+    Mainly useful when debugging this script.
+    """
+    dumped = tomllib.loads('\n'.join(manifest.dump()))
+    with filename.open('rb') as file:
+        from_file = tomllib.load(file)
+    if dumped != from_file:
+        print(f'Dump differs from loaded data!', file=sys.stderr)
+        diff = difflib.unified_diff(
+            pprint.pformat(dumped).splitlines(),
+            pprint.pformat(from_file).splitlines(),
+            '<dumped>', str(filename),
+            lineterm='',
+        )
+        for line in diff:
+            print(line, file=sys.stderr)
+        return False
+    else:
+        return True
+
 def main():
     parser = argparse.ArgumentParser(
         description=__doc__,
@@ -696,7 +684,16 @@ def main():
         run_all_generators = True
         args.unixy_check = True
 
-    with args.file.open() as file:
+    try:
+        file = args.file.open('rb')
+    except FileNotFoundError as err:
+        if args.file.suffix == '.txt':
+            # Provide a better error message
+            suggestion = args.file.with_suffix('.toml')
+            raise FileNotFoundError(
+                f'{args.file} not found. Did you mean {suggestion} ?') from err
+        raise
+    with file:
         manifest = parse_manifest(file)
 
     check_private_names(manifest)
@@ -709,7 +706,7 @@ def main():
     if args.dump:
         for line in manifest.dump():
             print(line)
-        results['dump'] = True
+        results['dump'] = check_dump(manifest, args.file)
 
     for gen in generators:
         filename = getattr(args, gen.var_name)