gh-113317: Argument Clinic: Add libclinic.clanguage (#117455)

Add libclinic.clanguage module and move the following classes and
functions there:

* CLanguage
* declare_parser()

Add libclinic.codegen and move the following classes there:

* BlockPrinter
* BufferSeries
* Destination

Move the following functions to libclinic.function:

* permute_left_option_groups()
* permute_optional_groups()
* permute_right_option_groups()

Lib/test/test_clinic.py

@@ -18,6 +18,9 @@ test_tools.skip_if_missing('clinic')
 with test_tools.imports_under_tool('clinic'):
     import libclinic
     from libclinic.converters import int_converter, str_converter
+    from libclinic.function import (
+        permute_optional_groups, permute_right_option_groups,
+        permute_left_option_groups)
     import clinic
     from clinic import DSLParser
 
@@ -679,7 +682,7 @@ class ParseFileUnitTest(TestCase):
 
 class ClinicGroupPermuterTest(TestCase):
     def _test(self, l, m, r, output):
-        computed = clinic.permute_optional_groups(l, m, r)
+        computed = permute_optional_groups(l, m, r)
         self.assertEqual(output, computed)
 
     def test_range(self):
@@ -721,7 +724,7 @@ class ClinicGroupPermuterTest(TestCase):
 
     def test_have_left_options_but_required_is_empty(self):
         def fn():
-            clinic.permute_optional_groups(['a'], [], [])
+            permute_optional_groups(['a'], [], [])
         self.assertRaises(ValueError, fn)
 
 
@@ -3764,7 +3767,7 @@ class PermutationTests(unittest.TestCase):
             (1, 2, 3),
         )
         data = list(zip([1, 2, 3]))  # Generate a list of 1-tuples.
-        actual = tuple(clinic.permute_left_option_groups(data))
+        actual = tuple(permute_left_option_groups(data))
         self.assertEqual(actual, expected)
 
     def test_permute_right_option_groups(self):
@@ -3775,7 +3778,7 @@ class PermutationTests(unittest.TestCase):
             (1, 2, 3),
         )
         data = list(zip([1, 2, 3]))  # Generate a list of 1-tuples.
-        actual = tuple(clinic.permute_right_option_groups(data))
+        actual = tuple(permute_right_option_groups(data))
         self.assertEqual(actual, expected)
 
     def test_permute_optional_groups(self):
@@ -3854,7 +3857,7 @@ class PermutationTests(unittest.TestCase):
         for params in dataset:
             with self.subTest(**params):
                 left, required, right, expected = params.values()
-                permutations = clinic.permute_optional_groups(left, required, right)
+                permutations = permute_optional_groups(left, required, right)
                 actual = tuple(permutations)
                 self.assertEqual(actual, expected)
 

Tools/clinic/libclinic/codegen.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+import dataclasses as dc
+import io
+import os
+from typing import Final, TYPE_CHECKING
+if TYPE_CHECKING:
+    from clinic import Clinic
+
+import libclinic
+from libclinic import fail
+from libclinic.crenderdata import Include
+from libclinic.language import Language
+from libclinic.block_parser import Block
+
+
+@dc.dataclass(slots=True)
+class BlockPrinter:
+    language: Language
+    f: io.StringIO = dc.field(default_factory=io.StringIO)
+
+    # '#include "header.h"   // reason': column of '//' comment
+    INCLUDE_COMMENT_COLUMN: Final[int] = 35
+
+    def print_block(
+        self,
+        block: Block,
+        *,
+        core_includes: bool = False,
+        limited_capi: bool,
+        header_includes: dict[str, Include],
+    ) -> None:
+        input = block.input
+        output = block.output
+        dsl_name = block.dsl_name
+        write = self.f.write
+
+        assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name)
+
+        if not dsl_name:
+            write(input)
+            return
+
+        write(self.language.start_line.format(dsl_name=dsl_name))
+        write("\n")
+
+        body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)
+        if not body_prefix:
+            write(input)
+        else:
+            for line in input.split('\n'):
+                write(body_prefix)
+                write(line)
+                write("\n")
+
+        write(self.language.stop_line.format(dsl_name=dsl_name))
+        write("\n")
+
+        output = ''
+        if core_includes and header_includes:
+            # Emit optional "#include" directives for C headers
+            output += '\n'
+
+            current_condition: str | None = None
+            includes = sorted(header_includes.values(), key=Include.sort_key)
+            for include in includes:
+                if include.condition != current_condition:
+                    if current_condition:
+                        output += '#endif\n'
+                    current_condition = include.condition
+                    if include.condition:
+                        output += f'{include.condition}\n'
+
+                if current_condition:
+                    line = f'#  include "{include.filename}"'
+                else:
+                    line = f'#include "{include.filename}"'
+                if include.reason:
+                    comment = f'// {include.reason}\n'
+                    line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment
+                output += line
+
+            if current_condition:
+                output += '#endif\n'
+
+        input = ''.join(block.input)
+        output += ''.join(block.output)
+        if output:
+            if not output.endswith('\n'):
+                output += '\n'
+            write(output)
+
+        arguments = "output={output} input={input}".format(
+            output=libclinic.compute_checksum(output, 16),
+            input=libclinic.compute_checksum(input, 16)
+        )
+        write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments))
+        write("\n")
+
+    def write(self, text: str) -> None:
+        self.f.write(text)
+
+
+class BufferSeries:
+    """
+    Behaves like a "defaultlist".
+    When you ask for an index that doesn't exist yet,
+    the object grows the list until that item exists.
+    So o[n] will always work.
+
+    Supports negative indices for actual items.
+    e.g. o[-1] is an element immediately preceding o[0].
+    """
+
+    def __init__(self) -> None:
+        self._start = 0
+        self._array: list[list[str]] = []
+
+    def __getitem__(self, i: int) -> list[str]:
+        i -= self._start
+        if i < 0:
+            self._start += i
+            prefix: list[list[str]] = [[] for x in range(-i)]
+            self._array = prefix + self._array
+            i = 0
+        while i >= len(self._array):
+            self._array.append([])
+        return self._array[i]
+
+    def clear(self) -> None:
+        for ta in self._array:
+            ta.clear()
+
+    def dump(self) -> str:
+        texts = ["".join(ta) for ta in self._array]
+        self.clear()
+        return "".join(texts)
+
+
+@dc.dataclass(slots=True, repr=False)
+class Destination:
+    name: str
+    type: str
+    clinic: Clinic
+    buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries)
+    filename: str = dc.field(init=False)  # set in __post_init__
+
+    args: dc.InitVar[tuple[str, ...]] = ()
+
+    def __post_init__(self, args: tuple[str, ...]) -> None:
+        valid_types = ('buffer', 'file', 'suppress')
+        if self.type not in valid_types:
+            fail(
+                f"Invalid destination type {self.type!r} for {self.name}, "
+                f"must be {', '.join(valid_types)}"
+            )
+        extra_arguments = 1 if self.type == "file" else 0
+        if len(args) < extra_arguments:
+            fail(f"Not enough arguments for destination "
+                 f"{self.name!r} new {self.type!r}")
+        if len(args) > extra_arguments:
+            fail(f"Too many arguments for destination {self.name!r} new {self.type!r}")
+        if self.type =='file':
+            d = {}
+            filename = self.clinic.filename
+            d['path'] = filename
+            dirname, basename = os.path.split(filename)
+            if not dirname:
+                dirname = '.'
+            d['dirname'] = dirname
+            d['basename'] = basename
+            d['basename_root'], d['basename_extension'] = os.path.splitext(filename)
+            self.filename = args[0].format_map(d)
+
+    def __repr__(self) -> str:
+        if self.type == 'file':
+            type_repr = f"type='file' file={self.filename!r}"
+        else:
+            type_repr = f"type={self.type!r}"
+        return f"<clinic.Destination {self.name!r} {type_repr}>"
+
+    def clear(self) -> None:
+        if self.type != 'buffer':
+            fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'")
+        self.buffers.clear()
+
+    def dump(self) -> str:
+        return self.buffers.dump()

Tools/clinic/libclinic/function.py

@@ -4,6 +4,7 @@ import copy
 import enum
 import functools
 import inspect
+from collections.abc import Iterable, Iterator, Sequence
 from typing import Final, Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from clinic import Clinic
@@ -238,3 +239,73 @@ class Parameter:
         lines = [f"  {self.name}"]
         lines.extend(f"    {line}" for line in self.docstring.split("\n"))
         return "\n".join(lines).rstrip()
+
+
+ParamTuple = tuple["Parameter", ...]
+
+
+def permute_left_option_groups(
+    l: Sequence[Iterable[Parameter]]
+) -> Iterator[ParamTuple]:
+    """
+    Given [(1,), (2,), (3,)], should yield:
+       ()
+       (3,)
+       (2, 3)
+       (1, 2, 3)
+    """
+    yield tuple()
+    accumulator: list[Parameter] = []
+    for group in reversed(l):
+        accumulator = list(group) + accumulator
+        yield tuple(accumulator)
+
+
+def permute_right_option_groups(
+    l: Sequence[Iterable[Parameter]]
+) -> Iterator[ParamTuple]:
+    """
+    Given [(1,), (2,), (3,)], should yield:
+      ()
+      (1,)
+      (1, 2)
+      (1, 2, 3)
+    """
+    yield tuple()
+    accumulator: list[Parameter] = []
+    for group in l:
+        accumulator.extend(group)
+        yield tuple(accumulator)
+
+
+def permute_optional_groups(
+    left: Sequence[Iterable[Parameter]],
+    required: Iterable[Parameter],
+    right: Sequence[Iterable[Parameter]]
+) -> tuple[ParamTuple, ...]:
+    """
+    Generator function that computes the set of acceptable
+    argument lists for the provided iterables of
+    argument groups.  (Actually it generates a tuple of tuples.)
+
+    Algorithm: prefer left options over right options.
+
+    If required is empty, left must also be empty.
+    """
+    required = tuple(required)
+    if not required:
+        if left:
+            raise ValueError("required is empty but left is not")
+
+    accumulator: list[ParamTuple] = []
+    counts = set()
+    for r in permute_right_option_groups(right):
+        for l in permute_left_option_groups(left):
+            t = l + required + r
+            if len(t) in counts:
+                continue
+            counts.add(len(t))
+            accumulator.append(t)
+
+    accumulator.sort(key=len)
+    return tuple(accumulator)