mirror of
https://github.com/python/cpython.git
synced 2025-08-02 16:13:13 +00:00
e55a0e971b
Currently, empty sequences in gather rules make the conditional for gather rules fail as empty sequences evaluate as "False". We need to explicitly check for "None" (the failure condition) to avoid false negatives.
241 lines
8.3 KiB
Python
241 lines
8.3 KiB
Python
import token
|
|
from typing import Any, Dict, Optional, IO, Text, Tuple
|
|
|
|
from pegen.grammar import (
|
|
Cut,
|
|
GrammarVisitor,
|
|
NameLeaf,
|
|
StringLeaf,
|
|
Rhs,
|
|
NamedItem,
|
|
Lookahead,
|
|
PositiveLookahead,
|
|
NegativeLookahead,
|
|
Opt,
|
|
Repeat0,
|
|
Repeat1,
|
|
Gather,
|
|
Group,
|
|
Rule,
|
|
Alt,
|
|
)
|
|
from pegen import grammar
|
|
from pegen.parser_generator import ParserGenerator
|
|
|
|
MODULE_PREFIX = """\
|
|
#!/usr/bin/env python3.8
|
|
# @generated by pegen from {filename}
|
|
|
|
import ast
|
|
import sys
|
|
import tokenize
|
|
|
|
from typing import Any, Optional
|
|
|
|
from pegen.parser import memoize, memoize_left_rec, logger, Parser
|
|
|
|
"""
|
|
MODULE_SUFFIX = """
|
|
|
|
if __name__ == '__main__':
|
|
from pegen.parser import simple_parser_main
|
|
simple_parser_main(GeneratedParser)
|
|
"""
|
|
|
|
|
|
class PythonCallMakerVisitor(GrammarVisitor):
|
|
def __init__(self, parser_generator: ParserGenerator):
|
|
self.gen = parser_generator
|
|
self.cache: Dict[Any, Any] = {}
|
|
|
|
def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]:
|
|
name = node.value
|
|
if name in ("NAME", "NUMBER", "STRING", "OP"):
|
|
name = name.lower()
|
|
return name, f"self.{name}()"
|
|
if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"):
|
|
return name.lower(), f"self.expect({name!r})"
|
|
return name, f"self.{name}()"
|
|
|
|
def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]:
|
|
return "literal", f"self.expect({node.value})"
|
|
|
|
def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]:
|
|
if node in self.cache:
|
|
return self.cache[node]
|
|
if len(node.alts) == 1 and len(node.alts[0].items) == 1:
|
|
self.cache[node] = self.visit(node.alts[0].items[0])
|
|
else:
|
|
name = self.gen.name_node(node)
|
|
self.cache[node] = name, f"self.{name}()"
|
|
return self.cache[node]
|
|
|
|
def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]:
|
|
name, call = self.visit(node.item)
|
|
if node.name:
|
|
name = node.name
|
|
return name, call
|
|
|
|
def lookahead_call_helper(self, node: Lookahead) -> Tuple[str, str]:
|
|
name, call = self.visit(node.node)
|
|
head, tail = call.split("(", 1)
|
|
assert tail[-1] == ")"
|
|
tail = tail[:-1]
|
|
return head, tail
|
|
|
|
def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]:
|
|
head, tail = self.lookahead_call_helper(node)
|
|
return None, f"self.positive_lookahead({head}, {tail})"
|
|
|
|
def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]:
|
|
head, tail = self.lookahead_call_helper(node)
|
|
return None, f"self.negative_lookahead({head}, {tail})"
|
|
|
|
def visit_Opt(self, node: Opt) -> Tuple[str, str]:
|
|
name, call = self.visit(node.node)
|
|
# Note trailing comma (the call may already have one comma
|
|
# at the end, for example when rules have both repeat0 and optional
|
|
# markers, e.g: [rule*])
|
|
if call.endswith(","):
|
|
return "opt", call
|
|
else:
|
|
return "opt", f"{call},"
|
|
|
|
def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]:
|
|
if node in self.cache:
|
|
return self.cache[node]
|
|
name = self.gen.name_loop(node.node, False)
|
|
self.cache[node] = name, f"self.{name}()," # Also a trailing comma!
|
|
return self.cache[node]
|
|
|
|
def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]:
|
|
if node in self.cache:
|
|
return self.cache[node]
|
|
name = self.gen.name_loop(node.node, True)
|
|
self.cache[node] = name, f"self.{name}()" # But no trailing comma here!
|
|
return self.cache[node]
|
|
|
|
def visit_Gather(self, node: Gather) -> Tuple[str, str]:
|
|
if node in self.cache:
|
|
return self.cache[node]
|
|
name = self.gen.name_gather(node)
|
|
self.cache[node] = name, f"self.{name}()" # No trailing comma here either!
|
|
return self.cache[node]
|
|
|
|
def visit_Group(self, node: Group) -> Tuple[Optional[str], str]:
|
|
return self.visit(node.rhs)
|
|
|
|
def visit_Cut(self, node: Cut) -> Tuple[str, str]:
|
|
return "cut", "True"
|
|
|
|
|
|
class PythonParserGenerator(ParserGenerator, GrammarVisitor):
|
|
def __init__(
|
|
self,
|
|
grammar: grammar.Grammar,
|
|
file: Optional[IO[Text]],
|
|
tokens: Dict[int, str] = token.tok_name,
|
|
):
|
|
super().__init__(grammar, tokens, file)
|
|
self.callmakervisitor = PythonCallMakerVisitor(self)
|
|
|
|
def generate(self, filename: str) -> None:
|
|
header = self.grammar.metas.get("header", MODULE_PREFIX)
|
|
if header is not None:
|
|
self.print(header.rstrip("\n").format(filename=filename))
|
|
subheader = self.grammar.metas.get("subheader", "")
|
|
if subheader:
|
|
self.print(subheader.format(filename=filename))
|
|
self.print("class GeneratedParser(Parser):")
|
|
while self.todo:
|
|
for rulename, rule in list(self.todo.items()):
|
|
del self.todo[rulename]
|
|
self.print()
|
|
with self.indent():
|
|
self.visit(rule)
|
|
trailer = self.grammar.metas.get("trailer", MODULE_SUFFIX)
|
|
if trailer is not None:
|
|
self.print(trailer.rstrip("\n"))
|
|
|
|
def visit_Rule(self, node: Rule) -> None:
|
|
is_loop = node.is_loop()
|
|
is_gather = node.is_gather()
|
|
rhs = node.flatten()
|
|
if node.left_recursive:
|
|
if node.leader:
|
|
self.print("@memoize_left_rec")
|
|
else:
|
|
# Non-leader rules in a cycle are not memoized,
|
|
# but they must still be logged.
|
|
self.print("@logger")
|
|
else:
|
|
self.print("@memoize")
|
|
node_type = node.type or "Any"
|
|
self.print(f"def {node.name}(self) -> Optional[{node_type}]:")
|
|
with self.indent():
|
|
self.print(f"# {node.name}: {rhs}")
|
|
if node.nullable:
|
|
self.print(f"# nullable={node.nullable}")
|
|
self.print("mark = self.mark()")
|
|
if is_loop:
|
|
self.print("children = []")
|
|
self.visit(rhs, is_loop=is_loop, is_gather=is_gather)
|
|
if is_loop:
|
|
self.print("return children")
|
|
else:
|
|
self.print("return None")
|
|
|
|
def visit_NamedItem(self, node: NamedItem) -> None:
|
|
name, call = self.callmakervisitor.visit(node.item)
|
|
if node.name:
|
|
name = node.name
|
|
if not name:
|
|
self.print(call)
|
|
else:
|
|
if name != "cut":
|
|
name = self.dedupe(name)
|
|
self.print(f"({name} := {call})")
|
|
|
|
def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None:
|
|
if is_loop:
|
|
assert len(node.alts) == 1
|
|
for alt in node.alts:
|
|
self.visit(alt, is_loop=is_loop, is_gather=is_gather)
|
|
|
|
def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None:
|
|
with self.local_variable_context():
|
|
self.print("cut = False") # TODO: Only if needed.
|
|
if is_loop:
|
|
self.print("while (")
|
|
else:
|
|
self.print("if (")
|
|
with self.indent():
|
|
first = True
|
|
for item in node.items:
|
|
if first:
|
|
first = False
|
|
else:
|
|
self.print("and")
|
|
self.visit(item)
|
|
if is_gather:
|
|
self.print("is not None")
|
|
|
|
self.print("):")
|
|
with self.indent():
|
|
action = node.action
|
|
if not action:
|
|
if is_gather:
|
|
assert len(self.local_variable_names) == 2
|
|
action = (
|
|
f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}"
|
|
)
|
|
else:
|
|
action = f"[{', '.join(self.local_variable_names)}]"
|
|
if is_loop:
|
|
self.print(f"children.append({action})")
|
|
self.print(f"mark = self.mark()")
|
|
else:
|
|
self.print(f"return {action}")
|
|
self.print("self.reset(mark)")
|
|
# Skip remaining alternatives if a cut was reached.
|
|
self.print("if cut: return None") # TODO: Only if needed.
|