mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 16:39:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-113317: Add Codegen class to Argument Clinic (#117626)

Browse source 
* Move ifndef_symbols, includes and add_include() from Clinic to
  Codegen. Add a 'codegen' (Codegen) attribute to Clinic.
* Remove libclinic.crenderdata module: move code to libclinic.codegen.
* BlockPrinter.print_block(): remove unused 'limited_capi' argument.
  Remove also 'core_includes' parameter.
* Add get_includes() methods.
* Make Codegen.ifndef_symbols private.
* Make Codegen.includes private.
* Make CConverter.includes private.
This commit is contained in:
Victor Stinner 2024-04-11 12:15:48 +02:00 • committed by GitHub
parent d4963871b0
commit a2ae84726b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 183 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Lib/test/test_clinic.py
View file

@ -877,9 +877,8 @@ class ClinicBlockParserTest(TestCase):
blocks = list(BlockParser(input, language))
writer = BlockPrinter(language)
c = _make_clinic()
for block in blocks:
writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes)
writer.print_block(block)
output = writer.f.getvalue()
assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input)

40
Tools/clinic/libclinic/app.py
View file

@ -9,8 +9,7 @@ import libclinic
from libclinic import fail, warn
from libclinic.function import Class
from libclinic.block_parser import Block, BlockParser
from libclinic.crenderdata import Include
from libclinic.codegen import BlockPrinter, Destination
from libclinic.codegen import BlockPrinter, Destination, Codegen
from libclinic.parser import Parser, PythonParser
from libclinic.dsl_parser import DSLParser
if TYPE_CHECKING:
@ -102,8 +101,7 @@ impl_definition block
self.modules: ModuleDict = {}
self.classes: ClassDict = {}
self.functions: list[Function] = []
# dict: include name => Include instance
self.includes: dict[str, Include] = {}
self.codegen = Codegen(self.limited_capi)
self.line_prefix = self.line_suffix = ''
@ -132,7 +130,6 @@ impl_definition block
DestBufferList = list[DestBufferType]
self.destination_buffers_stack: DestBufferList = []
self.ifndef_symbols: set[str] = set()
self.presets: dict[str, dict[Any, Any]] = {}
preset = None
@ -159,24 +156,6 @@ impl_definition block
assert name in self.destination_buffers
preset[name] = buffer
def add_include(self, name: str, reason: str,
*, condition: str | None = None) -> None:
try:
existing = self.includes[name]
except KeyError:
pass
else:
if existing.condition and not condition:
# If the previous include has a condition and the new one is
# unconditional, override the include.
pass
else:
# Already included, do nothing. Only mention a single reason,
# no need to list all of them.
return
self.includes[name] = Include(name, reason, condition)
def add_destination(
self,
name: str,
@ -212,9 +191,7 @@ impl_definition block
self.parsers[dsl_name] = parsers[dsl_name](self)
parser = self.parsers[dsl_name]
parser.parse(block)
printer.print_block(block,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer.print_block(block)
# these are destinations not buffers
for name, destination in self.destinations.items():
@ -229,9 +206,7 @@ impl_definition block
block.input = "dump " + name + "\n"
warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.")
printer.write("\n")
printer.print_block(block,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer.print_block(block)
continue
if destination.type == 'file':
@ -255,11 +230,10 @@ impl_definition block
pass
block.input = 'preserve\n'
includes = self.codegen.get_includes()
printer_2 = BlockPrinter(self.language)
printer_2.print_block(block,
core_includes=True,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer_2.print_block(block, header_includes=includes)
libclinic.write_file(destination.filename,
printer_2.f.getvalue())
continue

100
Tools/clinic/libclinic/clanguage.py
View file

@ -11,7 +11,7 @@ from libclinic import (
unspecified, fail, warn, Sentinels, VersionTuple)
from libclinic.function import (
GETTER, SETTER, METHOD_INIT, METHOD_NEW)
from libclinic.crenderdata import CRenderData, TemplateDict
from libclinic.codegen import CRenderData, TemplateDict, Codegen
from libclinic.language import Language
from libclinic.function import (
Module, Class, Function, Parameter,
@ -26,8 +26,7 @@ def declare_parser(
f: Function,
*,
hasformat: bool = False,
clinic: Clinic,
limited_capi: bool,
codegen: Codegen,
) -> str:
"""
Generates the code template for a static local PyArg_Parser variable,
@ -35,6 +34,7 @@ def declare_parser(
kwtuple field is also statically initialized. Otherwise
it is initialized at runtime.
"""
limited_capi = codegen.limited_capi
if hasformat:
fname = ''
format_ = '.format = "{format_units}:{name}",'
@ -80,8 +80,8 @@ def declare_parser(
""" % num_keywords
condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)'
clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition)
clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition)
codegen.add_include('pycore_gc.h', 'PyGC_Head', condition=condition)
codegen.add_include('pycore_runtime.h', '_Py_ID()', condition=condition)
declarations += """
static const char * const _keywords[] = {{{keywords_c} NULL}};
@ -317,14 +317,14 @@ class CLanguage(Language):
self,
func: Function,
params: dict[int, Parameter],
argname_fmt: str | None,
argname_fmt: str | None = None,
*,
fastcall: bool,
limited_capi: bool,
clinic: Clinic,
codegen: Codegen,
) -> str:
assert len(params) > 0
last_param = next(reversed(params.values()))
limited_capi = codegen.limited_capi
# Format the deprecation message.
containscheck = ""
@ -336,11 +336,11 @@ class CLanguage(Language):
elif fastcall:
conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))")
containscheck = "PySequence_Contains"
clinic.add_include('pycore_runtime.h', '_Py_ID()')
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))")
containscheck = "PyDict_Contains"
clinic.add_include('pycore_runtime.h', '_Py_ID()')
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
conditions = [f"nargs < {i+1}"]
condition = ") || (".join(conditions)
@ -399,7 +399,7 @@ class CLanguage(Language):
def output_templates(
self,
f: Function,
clinic: Clinic
codegen: Codegen,
) -> dict[str, str]:
parameters = list(f.parameters.values())
assert parameters
@ -412,7 +412,7 @@ class CLanguage(Language):
converters = [p.converter for p in parameters]
if f.critical_section:
clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()')
codegen.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()')
has_option_groups = parameters and (parameters[0].group or parameters[-1].group)
simple_return = (f.return_converter.type == 'PyObject *'
and not f.critical_section)
@ -517,7 +517,7 @@ class CLanguage(Language):
parser_declarations=declarations)
fastcall = not new_or_init
limited_capi = clinic.limited_capi
limited_capi = codegen.limited_capi
if limited_capi and (pseudo_args or
(any(p.is_optional() for p in parameters) and
any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or
@ -673,8 +673,8 @@ class CLanguage(Language):
""",
indent=4))
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_CheckPositional()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_CheckPositional()')
parser_code = [libclinic.normalize_snippet(f"""
if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{
goto exit;
@ -735,8 +735,8 @@ class CLanguage(Language):
if limited_capi:
fastcall = False
if fastcall:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseStack()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseStack()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}",
{parse_arguments})) {{
@ -773,8 +773,8 @@ class CLanguage(Language):
fastcall = False
else:
if vararg == self.NO_VARARG:
clinic.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywords()')
args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % (
min_pos,
max_pos,
@ -782,8 +782,8 @@ class CLanguage(Language):
)
nargs = "nargs"
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywordsWithVararg()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywordsWithVararg()')
args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % (
min_pos,
max_pos,
@ -796,8 +796,7 @@ class CLanguage(Language):
flags = "METH_FASTCALL|METH_KEYWORDS"
parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS
argname_fmt = 'args[%d]'
declarations = declare_parser(f, clinic=clinic,
limited_capi=clinic.limited_capi)
declarations = declare_parser(f, codegen=codegen)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
if has_optional_kw:
declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only)
@ -812,8 +811,7 @@ class CLanguage(Language):
flags = "METH_VARARGS|METH_KEYWORDS"
parser_prototype = self.PARSER_PROTOTYPE_KEYWORD
argname_fmt = 'fastargs[%d]'
declarations = declare_parser(f, clinic=clinic,
limited_capi=clinic.limited_capi)
declarations = declare_parser(f, codegen=codegen)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
declarations += "\nPyObject * const *fastargs;"
declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
@ -832,10 +830,10 @@ class CLanguage(Language):
if parser_code is not None:
if deprecated_keywords:
code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt,
clinic=clinic,
fastcall=fastcall,
limited_capi=limited_capi)
code = self.deprecate_keyword_use(f, deprecated_keywords,
argname_fmt,
codegen=codegen,
fastcall=fastcall)
parser_code.append(code)
add_label: str | None = None
@ -903,9 +901,8 @@ class CLanguage(Language):
for parameter in parameters:
parameter.converter.use_converter()
declarations = declare_parser(f, clinic=clinic,
hasformat=True,
limited_capi=limited_capi)
declarations = declare_parser(f, codegen=codegen,
hasformat=True)
if limited_capi:
# positional-or-keyword arguments
assert not fastcall
@ -921,8 +918,8 @@ class CLanguage(Language):
declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);"
elif fastcall:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseStackAndKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseStackAndKeywords()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma}
{parse_arguments})) {{
@ -930,8 +927,8 @@ class CLanguage(Language):
}}
""", indent=4)]
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseTupleAndKeywordsFast()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseTupleAndKeywordsFast()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
{parse_arguments})) {{
@ -941,10 +938,9 @@ class CLanguage(Language):
if deprecated_positionals or deprecated_keywords:
declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
if deprecated_keywords:
code = self.deprecate_keyword_use(f, deprecated_keywords, None,
clinic=clinic,
fastcall=fastcall,
limited_capi=limited_capi)
code = self.deprecate_keyword_use(f, deprecated_keywords,
codegen=codegen,
fastcall=fastcall)
parser_code.append(code)
if deprecated_positionals:
@ -960,9 +956,9 @@ class CLanguage(Language):
# Copy includes from parameters to Clinic after parse_arg() has been
# called above.
for converter in converters:
for include in converter.includes:
clinic.add_include(include.filename, include.reason,
condition=include.condition)
for include in converter.get_includes():
codegen.add_include(include.filename, include.reason,
condition=include.condition)
if new_or_init:
methoddef_define = ''
@ -984,16 +980,16 @@ class CLanguage(Language):
if not parses_keywords:
declarations = '{base_type_ptr}'
clinic.add_include('pycore_modsupport.h',
'_PyArg_NoKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_NoKeywords()')
fields.insert(0, libclinic.normalize_snippet("""
if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{
goto exit;
}}
""", indent=4))
if not parses_positional:
clinic.add_include('pycore_modsupport.h',
'_PyArg_NoPositional()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_NoPositional()')
fields.insert(0, libclinic.normalize_snippet("""
if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{
goto exit;
@ -1030,8 +1026,7 @@ class CLanguage(Language):
cpp_if = "#if " + conditional
cpp_endif = "#endif /* " + conditional + " */"
if methoddef_define and f.full_name not in clinic.ifndef_symbols:
clinic.ifndef_symbols.add(f.full_name)
if methoddef_define and codegen.add_ifndef_symbol(f.full_name):
methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF
# add ';' to the end of parser_prototype and impl_prototype
@ -1190,16 +1185,17 @@ class CLanguage(Language):
clinic: Clinic,
f: Function | None
) -> str:
if f is None or clinic is None:
if f is None:
return ""
codegen = clinic.codegen
data = CRenderData()
assert f.parameters, "We should always have a 'self' at this point!"
parameters = f.render_parameters
converters = [p.converter for p in parameters]
templates = self.output_templates(f, clinic)
templates = self.output_templates(f, codegen)
f_self = parameters[0]
selfless = parameters[1:]
@ -1323,7 +1319,7 @@ class CLanguage(Language):
if has_option_groups:
self.render_option_group_parsing(f, template_dict,
limited_capi=clinic.limited_capi)
limited_capi=codegen.limited_capi)
# buffers, not destination
for name, destination in clinic.destination_buffers.items():

126
Tools/clinic/libclinic/codegen.py
View file

@ -6,13 +6,92 @@ from typing import Final, TYPE_CHECKING
import libclinic
from libclinic import fail
from libclinic.crenderdata import Include
from libclinic.language import Language
from libclinic.block_parser import Block
if TYPE_CHECKING:
from libclinic.app import Clinic
TemplateDict = dict[str, str]
class CRenderData:
def __init__(self) -> None:
# The C statements to declare variables.
# Should be full lines with \n eol characters.
self.declarations: list[str] = []
# The C statements required to initialize the variables before the parse call.
# Should be full lines with \n eol characters.
self.initializers: list[str] = []
# The C statements needed to dynamically modify the values
# parsed by the parse call, before calling the impl.
self.modifications: list[str] = []
# The entries for the "keywords" array for PyArg_ParseTuple.
# Should be individual strings representing the names.
self.keywords: list[str] = []
# The "format units" for PyArg_ParseTuple.
# Should be individual strings that will get
self.format_units: list[str] = []
# The varargs arguments for PyArg_ParseTuple.
self.parse_arguments: list[str] = []
# The parameter declarations for the impl function.
self.impl_parameters: list[str] = []
# The arguments to the impl function at the time it's called.
self.impl_arguments: list[str] = []
# For return converters: the name of the variable that
# should receive the value returned by the impl.
self.return_value = "return_value"
# For return converters: the code to convert the return
# value from the parse function. This is also where
# you should check the _return_value for errors, and
# "goto exit" if there are any.
self.return_conversion: list[str] = []
self.converter_retval = "_return_value"
# The C statements required to do some operations
# after the end of parsing but before cleaning up.
# These operations may be, for example, memory deallocations which
# can only be done without any error happening during argument parsing.
self.post_parsing: list[str] = []
# The C statements required to clean up after the impl call.
self.cleanup: list[str] = []
# The C statements to generate critical sections (per-object locking).
self.lock: list[str] = []
self.unlock: list[str] = []
@dc.dataclass(slots=True, frozen=True)
class Include:
"""
An include like: #include "pycore_long.h" // _Py_ID()
"""
# Example: "pycore_long.h".
filename: str
# Example: "_Py_ID()".
reason: str
# None means unconditional include.
# Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)".
condition: str | None
def sort_key(self) -> tuple[str, str]:
# order: '#if' comes before 'NO_CONDITION'
return (self.condition or 'NO_CONDITION', self.filename)
@dc.dataclass(slots=True)
class BlockPrinter:
language: Language
@ -25,9 +104,7 @@ class BlockPrinter:
self,
block: Block,
*,
core_includes: bool = False,
limited_capi: bool,
header_includes: dict[str, Include],
header_includes: list[Include] | None = None,
) -> None:
input = block.input
output = block.output
@ -56,13 +133,12 @@ class BlockPrinter:
write("\n")
output = ''
if core_includes and header_includes:
if header_includes:
# Emit optional "#include" directives for C headers
output += '\n'
current_condition: str | None = None
includes = sorted(header_includes.values(), key=Include.sort_key)
for include in includes:
for include in header_includes:
if include.condition != current_condition:
if current_condition:
output += '#endif\n'
@ -188,3 +264,39 @@ class Destination:
DestinationDict = dict[str, Destination]
class Codegen:
def __init__(self, limited_capi: bool) -> None:
self.limited_capi = limited_capi
self._ifndef_symbols: set[str] = set()
# dict: include name => Include instance
self._includes: dict[str, Include] = {}
def add_ifndef_symbol(self, name: str) -> bool:
if name in self._ifndef_symbols:
return False
self._ifndef_symbols.add(name)
return True
def add_include(self, name: str, reason: str,
*, condition: str | None = None) -> None:
try:
existing = self._includes[name]
except KeyError:
pass
else:
if existing.condition and not condition:
# If the previous include has a condition and the new one is
# unconditional, override the include.
pass
else:
# Already included, do nothing. Only mention a single reason,
# no need to list all of them.
return
self._includes[name] = Include(name, reason, condition)
def get_includes(self) -> list[Include]:
return sorted(self._includes.values(),
key=Include.sort_key)

9
Tools/clinic/libclinic/converter.py
View file

@ -7,7 +7,7 @@ from collections.abc import Callable
import libclinic
from libclinic import fail
from libclinic import Sentinels, unspecified, unknown
from libclinic.crenderdata import CRenderData, Include, TemplateDict
from libclinic.codegen import CRenderData, Include, TemplateDict
from libclinic.function import Function, Parameter
@ -180,7 +180,7 @@ class CConverter(metaclass=CConverterAutoRegister):
self.name = libclinic.ensure_legal_c_identifier(name)
self.py_name = py_name
self.unused = unused
self.includes: list[Include] = []
self._includes: list[Include] = []
if default is not unspecified:
if (self.default_type
@ -513,7 +513,10 @@ class CConverter(metaclass=CConverterAutoRegister):
def add_include(self, name: str, reason: str,
*, condition: str | None = None) -> None:
include = Include(name, reason, condition)
self.includes.append(include)
self._includes.append(include)
def get_includes(self) -> list[Include]:
return self._includes
ConverterType = Callable[..., CConverter]

2
Tools/clinic/libclinic/converters.py
View file

@ -9,7 +9,7 @@ from libclinic.function import (
Function, Parameter,
CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW,
GETTER, SETTER)
from libclinic.crenderdata import CRenderData, TemplateDict
from libclinic.codegen import CRenderData, TemplateDict
from libclinic.converter import (
CConverter, legacy_converters, add_legacy_c_converter)

81
Tools/clinic/libclinic/crenderdata.py
View file

@ -1,81 +0,0 @@
import dataclasses as dc
TemplateDict = dict[str, str]
class CRenderData:
def __init__(self) -> None:
# The C statements to declare variables.
# Should be full lines with \n eol characters.
self.declarations: list[str] = []
# The C statements required to initialize the variables before the parse call.
# Should be full lines with \n eol characters.
self.initializers: list[str] = []
# The C statements needed to dynamically modify the values
# parsed by the parse call, before calling the impl.
self.modifications: list[str] = []
# The entries for the "keywords" array for PyArg_ParseTuple.
# Should be individual strings representing the names.
self.keywords: list[str] = []
# The "format units" for PyArg_ParseTuple.
# Should be individual strings that will get
self.format_units: list[str] = []
# The varargs arguments for PyArg_ParseTuple.
self.parse_arguments: list[str] = []
# The parameter declarations for the impl function.
self.impl_parameters: list[str] = []
# The arguments to the impl function at the time it's called.
self.impl_arguments: list[str] = []
# For return converters: the name of the variable that
# should receive the value returned by the impl.
self.return_value = "return_value"
# For return converters: the code to convert the return
# value from the parse function. This is also where
# you should check the _return_value for errors, and
# "goto exit" if there are any.
self.return_conversion: list[str] = []
self.converter_retval = "_return_value"
# The C statements required to do some operations
# after the end of parsing but before cleaning up.
# These operations may be, for example, memory deallocations which
# can only be done without any error happening during argument parsing.
self.post_parsing: list[str] = []
# The C statements required to clean up after the impl call.
self.cleanup: list[str] = []
# The C statements to generate critical sections (per-object locking).
self.lock: list[str] = []
self.unlock: list[str] = []
@dc.dataclass(slots=True, frozen=True)
class Include:
"""
An include like: #include "pycore_long.h" // _Py_ID()
"""
# Example: "pycore_long.h".
filename: str
# Example: "_Py_ID()".
reason: str
# None means unconditional include.
# Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)".
condition: str | None
def sort_key(self) -> tuple[str, str]:
# order: '#if' comes before 'NO_CONDITION'
return (self.condition or 'NO_CONDITION', self.filename)

2
Tools/clinic/libclinic/return_converters.py
View file

@ -1,6 +1,6 @@
import sys
from collections.abc import Callable
from libclinic.crenderdata import CRenderData
from libclinic.codegen import CRenderData
from libclinic.function import Function
from typing import Any