mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-07 21:29:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-113317: Argument Clinic: tear out internal text accumulator APIs (#113402)

Browse source 
Replace the internal accumulator APIs by using lists of strings and join().

Yak-shaving for separating out formatting code into a separate file.

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
Erlend E. Aasland 2023-12-22 21:28:15 +01:00 • committed by GitHub
parent a0d3d3ec9d
commit daa658aba5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 89 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

30
Lib/test/test_clinic.py
View file

@ -3761,20 +3761,6 @@ class FormatHelperTests(unittest.TestCase):
actual = clinic.normalize_snippet(snippet, indent=indent) actual = clinic.normalize_snippet(snippet, indent=indent)
self.assertEqual(actual, expected) self.assertEqual(actual, expected)
def test_accumulator(self):
acc = clinic.text_accumulator()
self.assertEqual(acc.output(), "")
acc.append("a")
self.assertEqual(acc.output(), "a")
self.assertEqual(acc.output(), "")
acc.append("b")
self.assertEqual(acc.output(), "b")
self.assertEqual(acc.output(), "")
acc.append("c")
acc.append("d")
self.assertEqual(acc.output(), "cd")
self.assertEqual(acc.output(), "")
def test_quoted_for_c_string(self): def test_quoted_for_c_string(self):
dataset = ( dataset = (
# input, expected # input, expected
@ -3790,22 +3776,6 @@ class FormatHelperTests(unittest.TestCase):
out = clinic.quoted_for_c_string(line) out = clinic.quoted_for_c_string(line)
self.assertEqual(out, expected) self.assertEqual(out, expected)
def test_rstrip_lines(self):
lines = (
"a \n"
"b\n"
" c\n"
" d \n"
)
expected = (
"a\n"
"b\n"
" c\n"
" d\n"
)
out = clinic.rstrip_lines(lines)
self.assertEqual(out, expected)
def test_format_escape(self): def test_format_escape(self):
line = "{}, {a}" line = "{}, {a}"
expected = "{{}}, {{a}}" expected = "{{}}, {{a}}"

247
Tools/clinic/clinic.py
View file

@ -105,42 +105,8 @@ NULL = Null()
sig_end_marker = '--' sig_end_marker = '--'
Appender = Callable[[str], None]
Outputter = Callable[[], str]
TemplateDict = dict[str, str] TemplateDict = dict[str, str]
class _TextAccumulator(NamedTuple):
text: list[str]
append: Appender
output: Outputter
def _text_accumulator() -> _TextAccumulator:
text: list[str] = []
def output() -> str:
s = ''.join(text)
text.clear()
return s
return _TextAccumulator(text, text.append, output)
class TextAccumulator(NamedTuple):
append: Appender
output: Outputter
def text_accumulator() -> TextAccumulator:
"""
Creates a simple text accumulator / joiner.
Returns a pair of callables:
append, output
"append" appends a string to the accumulator.
"output" returns the contents of the accumulator
joined together (''.join(accumulator)) and
empties the accumulator.
"""
text, append, output = _text_accumulator()
return TextAccumulator(append, output)
@dc.dataclass @dc.dataclass
class ClinicError(Exception): class ClinicError(Exception):
@ -264,14 +230,6 @@ def ensure_legal_c_identifier(s: str) -> str:
return s + "_value" return s + "_value"
return s return s
def rstrip_lines(s: str) -> str:
text, add, output = _text_accumulator()
for line in s.split('\n'):
add(line.rstrip())
add('\n')
text.pop()
return output()
def format_escape(s: str) -> str: def format_escape(s: str) -> str:
# double up curly-braces, this string will be used # double up curly-braces, this string will be used
# as part of a format_map() template later # as part of a format_map() template later
@ -294,18 +252,16 @@ def linear_format(s: str, **kwargs: str) -> str:
* A newline will be added to the end. * A newline will be added to the end.
""" """
add, output = text_accumulator() lines = []
for line in s.split('\n'): for line in s.split('\n'):
indent, curly, trailing = line.partition('{') indent, curly, trailing = line.partition('{')
if not curly: if not curly:
add(line) lines.extend([line, "\n"])
add('\n')
continue continue
name, curly, trailing = trailing.partition('}') name, curly, trailing = trailing.partition('}')
if not curly or name not in kwargs: if not curly or name not in kwargs:
add(line) lines.extend([line, "\n"])
add('\n')
continue continue
if trailing: if trailing:
@ -319,51 +275,32 @@ def linear_format(s: str, **kwargs: str) -> str:
if not value: if not value:
continue continue
value = textwrap.indent(rstrip_lines(value), indent) stripped = [line.rstrip() for line in value.split("\n")]
add(value) value = textwrap.indent("\n".join(stripped), indent)
add('\n') lines.extend([value, "\n"])
return output()[:-1] return "".join(lines[:-1])
def indent_all_lines(s: str, prefix: str) -> str: def _add_prefix_and_suffix(s: str, prefix: str = "", suffix: str = "") -> str:
""" """
Returns 's', with 'prefix' prepended to all lines. Return 's', with 'prefix' prepended and 'suffix' appended to all lines.
If the last line is empty, prefix is not prepended If the last line is empty, it remains unchanged.
to it. (If s is blank, returns s unchanged.) If s is blank, returns s unchanged.
(textwrap.indent only adds to non-blank lines.) (textwrap.indent only adds to non-blank lines.)
""" """
split = s.split('\n') *split, last = s.split("\n")
last = split.pop() lines = [prefix + line + suffix + "\n" for line in split]
final = []
for line in split:
final.append(prefix)
final.append(line)
final.append('\n')
if last: if last:
final.append(prefix) lines.append(prefix + last + suffix)
final.append(last) return "".join(lines)
return ''.join(final)
def indent_all_lines(s: str, prefix: str) -> str:
return _add_prefix_and_suffix(s, prefix=prefix)
def suffix_all_lines(s: str, suffix: str) -> str: def suffix_all_lines(s: str, suffix: str) -> str:
""" return _add_prefix_and_suffix(s, suffix=suffix)
Returns 's', with 'suffix' appended to all lines.
If the last line is empty, suffix is not appended
to it. (If s is blank, returns s unchanged.)
"""
split = s.split('\n')
last = split.pop()
final = []
for line in split:
final.append(line)
final.append(suffix)
final.append('\n')
if last:
final.append(last)
final.append(suffix)
return ''.join(final)
def pprint_words(items: list[str]) -> str: def pprint_words(items: list[str]) -> str:
@ -1068,21 +1005,21 @@ class CLanguage(Language):
self, self,
f: Function f: Function
) -> str: ) -> str:
text, add, output = _text_accumulator() lines = []
# turn docstring into a properly quoted C string # turn docstring into a properly quoted C string
for line in f.docstring.split('\n'): for line in f.docstring.split('\n'):
add('"') lines.append('"')
add(quoted_for_c_string(line)) lines.append(quoted_for_c_string(line))
add('\\n"\n') lines.append('\\n"\n')
if text[-2] == sig_end_marker: if lines[-2] == sig_end_marker:
# If we only have a signature, add the blank line that the # If we only have a signature, add the blank line that the
# __text_signature__ getter expects to be there. # __text_signature__ getter expects to be there.
add('"\\n"') lines.append('"\\n"')
else: else:
text.pop() lines.pop()
add('"') lines.append('"')
return ''.join(text) return ''.join(lines)
def output_templates( def output_templates(
self, self,
@ -1183,8 +1120,8 @@ class CLanguage(Language):
declarations: str = '' declarations: str = ''
) -> str: ) -> str:
nonlocal parser_body_fields nonlocal parser_body_fields
add, output = text_accumulator() lines = []
add(prototype) lines.append(prototype)
parser_body_fields = fields parser_body_fields = fields
preamble = normalize_snippet(""" preamble = normalize_snippet("""
@ -1208,9 +1145,8 @@ class CLanguage(Language):
}} }}
""") """)
for field in preamble, *fields, finale: for field in preamble, *fields, finale:
add('\n') lines.append(field)
add(field) return linear_format("\n".join(lines), parser_declarations=declarations)
return linear_format(output(), parser_declarations=declarations)
fastcall = not new_or_init fastcall = not new_or_init
limited_capi = clinic.limited_capi limited_capi = clinic.limited_capi
@ -1785,7 +1721,7 @@ class CLanguage(Language):
# Clinic prefers groups on the left. So in the above example, # Clinic prefers groups on the left. So in the above example,
# five arguments would map to B+C, not C+D. # five arguments would map to B+C, not C+D.
add, output = text_accumulator() out = []
parameters = list(f.parameters.values()) parameters = list(f.parameters.values())
if isinstance(parameters[0].converter, self_converter): if isinstance(parameters[0].converter, self_converter):
del parameters[0] del parameters[0]
@ -1817,14 +1753,14 @@ class CLanguage(Language):
nargs = 'PyTuple_Size(args)' nargs = 'PyTuple_Size(args)'
else: else:
nargs = 'PyTuple_GET_SIZE(args)' nargs = 'PyTuple_GET_SIZE(args)'
add(f"switch ({nargs}) {{\n") out.append(f"switch ({nargs}) {{\n")
for subset in permute_optional_groups(left, required, right): for subset in permute_optional_groups(left, required, right):
count = len(subset) count = len(subset)
count_min = min(count_min, count) count_min = min(count_min, count)
count_max = max(count_max, count) count_max = max(count_max, count)
if count == 0: if count == 0:
add(""" case 0: out.append(""" case 0:
break; break;
""") """)
continue continue
@ -1856,14 +1792,15 @@ class CLanguage(Language):
""" """
s = linear_format(s, group_booleans=lines) s = linear_format(s, group_booleans=lines)
s = s.format_map(d) s = s.format_map(d)
add(s) out.append(s)
add(" default:\n") out.append(" default:\n")
s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n'
add(s.format(f.full_name, count_min, count_max)) out.append(s.format(f.full_name, count_min, count_max))
add(' goto exit;\n') out.append(' goto exit;\n')
add("}") out.append("}")
template_dict['option_group_parsing'] = format_escape(output())
template_dict['option_group_parsing'] = format_escape("".join(out))
def render_function( def render_function(
self, self,
@ -1873,7 +1810,6 @@ class CLanguage(Language):
if f is None or clinic is None: if f is None or clinic is None:
return "" return ""
add, output = text_accumulator()
data = CRenderData() data = CRenderData()
assert f.parameters, "We should always have a 'self' at this point!" assert f.parameters, "We should always have a 'self' at this point!"
@ -2202,7 +2138,7 @@ class BlockParser:
return line return line
def parse_verbatim_block(self) -> Block: def parse_verbatim_block(self) -> Block:
add, output = text_accumulator() lines = []
self.block_start_line_number = self.line_number self.block_start_line_number = self.line_number
while self.input: while self.input:
@ -2211,12 +2147,12 @@ class BlockParser:
if dsl_name: if dsl_name:
self.dsl_name = dsl_name self.dsl_name = dsl_name
break break
add(line) lines.append(line)
return Block(output()) return Block("".join(lines))
def parse_clinic_block(self, dsl_name: str) -> Block: def parse_clinic_block(self, dsl_name: str) -> Block:
input_add, input_output = text_accumulator() in_lines = []
self.block_start_line_number = self.line_number + 1 self.block_start_line_number = self.line_number + 1
stop_line = self.language.stop_line.format(dsl_name=dsl_name) stop_line = self.language.stop_line.format(dsl_name=dsl_name)
body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)
@ -2244,7 +2180,7 @@ class BlockParser:
line = line.lstrip() line = line.lstrip()
assert line.startswith(body_prefix) assert line.startswith(body_prefix)
line = line.removeprefix(body_prefix) line = line.removeprefix(body_prefix)
input_add(line) in_lines.append(line)
# consume output and checksum line, if present. # consume output and checksum line, if present.
if self.last_dsl_name == dsl_name: if self.last_dsl_name == dsl_name:
@ -2258,7 +2194,7 @@ class BlockParser:
assert checksum_re is not None assert checksum_re is not None
# scan forward for checksum line # scan forward for checksum line
output_add, output_output = text_accumulator() out_lines = []
arguments = None arguments = None
while self.input: while self.input:
line = self._line(lookahead=True) line = self._line(lookahead=True)
@ -2266,12 +2202,12 @@ class BlockParser:
arguments = match.group(1) if match else None arguments = match.group(1) if match else None
if arguments: if arguments:
break break
output_add(line) out_lines.append(line)
if self.is_start_line(line): if self.is_start_line(line):
break break
output: str | None output: str | None
output = output_output() output = "".join(out_lines)
if arguments: if arguments:
d = {} d = {}
for field in shlex.split(arguments): for field in shlex.split(arguments):
@ -2299,7 +2235,7 @@ class BlockParser:
self.input.extend(reversed(output_lines)) self.input.extend(reversed(output_lines))
output = None output = None
return Block(input_output(), dsl_name, output=output) return Block("".join(in_lines), dsl_name, output=output)
@dc.dataclass(slots=True, frozen=True) @dc.dataclass(slots=True, frozen=True)
@ -2406,6 +2342,9 @@ class BlockPrinter:
self.f.write(text) self.f.write(text)
TextAccumulator = list[str]
class BufferSeries: class BufferSeries:
""" """
Behaves like a "defaultlist". Behaves like a "defaultlist".
@ -2419,26 +2358,26 @@ class BufferSeries:
def __init__(self) -> None: def __init__(self) -> None:
self._start = 0 self._start = 0
self._array: list[_TextAccumulator] = [] self._array: list[TextAccumulator] = []
self._constructor = _text_accumulator
def __getitem__(self, i: int) -> _TextAccumulator: def __getitem__(self, i: int) -> TextAccumulator:
i -= self._start i -= self._start
if i < 0: if i < 0:
self._start += i self._start += i
prefix = [self._constructor() for x in range(-i)] prefix: list[TextAccumulator] = [[] for x in range(-i)]
self._array = prefix + self._array self._array = prefix + self._array
i = 0 i = 0
while i >= len(self._array): while i >= len(self._array):
self._array.append(self._constructor()) self._array.append([])
return self._array[i] return self._array[i]
def clear(self) -> None: def clear(self) -> None:
for ta in self._array: for ta in self._array:
ta.text.clear() ta.clear()
def dump(self) -> str: def dump(self) -> str:
texts = [ta.output() for ta in self._array] texts = ["".join(ta) for ta in self._array]
self.clear()
return "".join(texts) return "".join(texts)
@ -2624,7 +2563,7 @@ impl_definition block
'impl_definition': d('block'), 'impl_definition': d('block'),
} }
DestBufferType = dict[str, _TextAccumulator] DestBufferType = dict[str, TextAccumulator]
DestBufferList = list[DestBufferType] DestBufferList = list[DestBufferType]
self.destination_buffers_stack: DestBufferList = [] self.destination_buffers_stack: DestBufferList = []
@ -2696,7 +2635,7 @@ impl_definition block
self, self,
name: str, name: str,
item: int = 0 item: int = 0
) -> _TextAccumulator: ) -> TextAccumulator:
d = self.get_destination(name) d = self.get_destination(name)
return d.buffers[item] return d.buffers[item]
@ -3150,11 +3089,9 @@ class Parameter:
return f'argument {i}' return f'argument {i}'
def render_docstring(self) -> str: def render_docstring(self) -> str:
add, out = text_accumulator() lines = [f" {self.name}"]
add(f" {self.name}\n") lines.extend(f" {line}" for line in self.docstring.split("\n"))
for line in self.docstring.split("\n"): return "\n".join(lines).rstrip()
add(f" {line}\n")
return out().rstrip()
CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"])
@ -6220,15 +6157,15 @@ class DSLParser:
def format_docstring_signature( def format_docstring_signature(
self, f: Function, parameters: list[Parameter] self, f: Function, parameters: list[Parameter]
) -> str: ) -> str:
text, add, output = _text_accumulator() lines = []
add(f.displayname) lines.append(f.displayname)
if self.forced_text_signature: if self.forced_text_signature:
add(self.forced_text_signature) lines.append(self.forced_text_signature)
elif f.kind in {GETTER, SETTER}: elif f.kind in {GETTER, SETTER}:
# @getter and @setter do not need signatures like a method or a function. # @getter and @setter do not need signatures like a method or a function.
return '' return ''
else: else:
add('(') lines.append('(')
# populate "right_bracket_count" field for every parameter # populate "right_bracket_count" field for every parameter
assert parameters, "We should always have a self parameter. " + repr(f) assert parameters, "We should always have a self parameter. " + repr(f)
@ -6282,7 +6219,7 @@ class DSLParser:
first_parameter = True first_parameter = True
last_p = parameters[-1] last_p = parameters[-1]
line_length = len(''.join(text)) line_length = len(''.join(lines))
indent = " " * line_length indent = " " * line_length
def add_parameter(text: str) -> None: def add_parameter(text: str) -> None:
nonlocal line_length nonlocal line_length
@ -6293,12 +6230,11 @@ class DSLParser:
else: else:
s = ' ' + text s = ' ' + text
if line_length + len(s) >= 72: if line_length + len(s) >= 72:
add('\n') lines.extend(["\n", indent])
add(indent)
line_length = len(indent) line_length = len(indent)
s = text s = text
line_length += len(s) line_length += len(s)
add(s) lines.append(s)
for p in parameters: for p in parameters:
if not p.converter.show_in_signature: if not p.converter.show_in_signature:
@ -6321,8 +6257,7 @@ class DSLParser:
added_star = True added_star = True
add_parameter('*,') add_parameter('*,')
p_add, p_output = text_accumulator() p_lines = [fix_right_bracket_count(p.right_bracket_count)]
p_add(fix_right_bracket_count(p.right_bracket_count))
if isinstance(p.converter, self_converter): if isinstance(p.converter, self_converter):
# annotate first parameter as being a "self". # annotate first parameter as being a "self".
@ -6340,30 +6275,31 @@ class DSLParser:
# have a docstring.) if this is an __init__ # have a docstring.) if this is an __init__
# (or __new__), then this signature is for # (or __new__), then this signature is for
# calling the class to construct a new instance. # calling the class to construct a new instance.
p_add('$') p_lines.append('$')
if p.is_vararg(): if p.is_vararg():
p_add("*") p_lines.append("*")
name = p.converter.signature_name or p.name name = p.converter.signature_name or p.name
p_add(name) p_lines.append(name)
if not p.is_vararg() and p.converter.is_optional(): if not p.is_vararg() and p.converter.is_optional():
p_add('=') p_lines.append('=')
value = p.converter.py_default value = p.converter.py_default
if not value: if not value:
value = repr(p.converter.default) value = repr(p.converter.default)
p_add(value) p_lines.append(value)
if (p != last_p) or need_a_trailing_slash: if (p != last_p) or need_a_trailing_slash:
p_add(',') p_lines.append(',')
add_parameter(p_output()) p_output = "".join(p_lines)
add_parameter(p_output)
add(fix_right_bracket_count(0)) lines.append(fix_right_bracket_count(0))
if need_a_trailing_slash: if need_a_trailing_slash:
add_parameter('/') add_parameter('/')
add(')') lines.append(')')
# PEP 8 says: # PEP 8 says:
# #
@ -6375,13 +6311,13 @@ class DSLParser:
# therefore this is commented out: # therefore this is commented out:
# #
# if f.return_converter.py_default: # if f.return_converter.py_default:
# add(' -> ') # lines.append(' -> ')
# add(f.return_converter.py_default) # lines.append(f.return_converter.py_default)
if not f.docstring_only: if not f.docstring_only:
add("\n" + sig_end_marker + "\n") lines.append("\n" + sig_end_marker + "\n")
signature_line = output() signature_line = "".join(lines)
# now fix up the places where the brackets look wrong # now fix up the places where the brackets look wrong
return signature_line.replace(', ]', ',] ') return signature_line.replace(', ]', ',] ')
@ -6389,12 +6325,7 @@ class DSLParser:
@staticmethod @staticmethod
def format_docstring_parameters(params: list[Parameter]) -> str: def format_docstring_parameters(params: list[Parameter]) -> str:
"""Create substitution text for {parameters}""" """Create substitution text for {parameters}"""
add, output = text_accumulator() return "".join(p.render_docstring() + "\n" for p in params if p.docstring)
for p in params:
if p.docstring:
add(p.render_docstring())
add('\n')
return output()
def format_docstring(self) -> str: def format_docstring(self) -> str:
assert self.function is not None assert self.function is not None