mirror of
https://github.com/python/cpython.git
synced 2025-11-04 03:44:55 +00:00
b01fd533fe
Simplify the peg generator logic by extracting as much visitors as possible to disentangle the flow and separate concerns.
364 lines
9 KiB
Python
364 lines
9 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import abstractmethod
|
|
from typing import (
|
|
TYPE_CHECKING,
|
|
AbstractSet,
|
|
Any,
|
|
Dict,
|
|
Iterable,
|
|
Iterator,
|
|
List,
|
|
Optional,
|
|
Set,
|
|
Tuple,
|
|
Union,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from pegen.parser_generator import ParserGenerator
|
|
|
|
|
|
class GrammarError(Exception):
|
|
pass
|
|
|
|
|
|
class GrammarVisitor:
|
|
def visit(self, node: Any, *args: Any, **kwargs: Any) -> Any:
|
|
"""Visit a node."""
|
|
method = "visit_" + node.__class__.__name__
|
|
visitor = getattr(self, method, self.generic_visit)
|
|
return visitor(node, *args, **kwargs)
|
|
|
|
def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Any:
|
|
"""Called if no explicit visitor function exists for a node."""
|
|
for value in node:
|
|
if isinstance(value, list):
|
|
for item in value:
|
|
self.visit(item, *args, **kwargs)
|
|
else:
|
|
self.visit(value, *args, **kwargs)
|
|
|
|
|
|
class Grammar:
|
|
def __init__(self, rules: Iterable[Rule], metas: Iterable[Tuple[str, Optional[str]]]):
|
|
self.rules = {rule.name: rule for rule in rules}
|
|
self.metas = dict(metas)
|
|
|
|
def __str__(self) -> str:
|
|
return "\n".join(str(rule) for name, rule in self.rules.items())
|
|
|
|
def __repr__(self) -> str:
|
|
lines = ["Grammar("]
|
|
lines.append(" [")
|
|
for rule in self.rules.values():
|
|
lines.append(f" {repr(rule)},")
|
|
lines.append(" ],")
|
|
lines.append(" {repr(list(self.metas.items()))}")
|
|
lines.append(")")
|
|
return "\n".join(lines)
|
|
|
|
def __iter__(self) -> Iterator[Rule]:
|
|
yield from self.rules.values()
|
|
|
|
|
|
# Global flag whether we want actions in __str__() -- default off.
|
|
SIMPLE_STR = True
|
|
|
|
|
|
class Rule:
|
|
def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[object] = None):
|
|
self.name = name
|
|
self.type = type
|
|
self.rhs = rhs
|
|
self.memo = bool(memo)
|
|
self.left_recursive = False
|
|
self.leader = False
|
|
|
|
def is_loop(self) -> bool:
|
|
return self.name.startswith("_loop")
|
|
|
|
def is_gather(self) -> bool:
|
|
return self.name.startswith("_gather")
|
|
|
|
def __str__(self) -> str:
|
|
if SIMPLE_STR or self.type is None:
|
|
res = f"{self.name}: {self.rhs}"
|
|
else:
|
|
res = f"{self.name}[{self.type}]: {self.rhs}"
|
|
if len(res) < 88:
|
|
return res
|
|
lines = [res.split(":")[0] + ":"]
|
|
lines += [f" | {alt}" for alt in self.rhs.alts]
|
|
return "\n".join(lines)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Rule({self.name!r}, {self.type!r}, {self.rhs!r})"
|
|
|
|
def __iter__(self) -> Iterator[Rhs]:
|
|
yield self.rhs
|
|
|
|
def flatten(self) -> Rhs:
|
|
# If it's a single parenthesized group, flatten it.
|
|
rhs = self.rhs
|
|
if (
|
|
not self.is_loop()
|
|
and len(rhs.alts) == 1
|
|
and len(rhs.alts[0].items) == 1
|
|
and isinstance(rhs.alts[0].items[0].item, Group)
|
|
):
|
|
rhs = rhs.alts[0].items[0].item.rhs
|
|
return rhs
|
|
|
|
|
|
class Leaf:
|
|
def __init__(self, value: str):
|
|
self.value = value
|
|
|
|
def __str__(self) -> str:
|
|
return self.value
|
|
|
|
def __iter__(self) -> Iterable[str]:
|
|
if False:
|
|
yield
|
|
|
|
|
|
class NameLeaf(Leaf):
|
|
"""The value is the name."""
|
|
|
|
def __str__(self) -> str:
|
|
if self.value == "ENDMARKER":
|
|
return "$"
|
|
return super().__str__()
|
|
|
|
def __repr__(self) -> str:
|
|
return f"NameLeaf({self.value!r})"
|
|
|
|
|
|
class StringLeaf(Leaf):
|
|
"""The value is a string literal, including quotes."""
|
|
|
|
def __repr__(self) -> str:
|
|
return f"StringLeaf({self.value!r})"
|
|
|
|
|
|
class Rhs:
|
|
def __init__(self, alts: List[Alt]):
|
|
self.alts = alts
|
|
self.memo: Optional[Tuple[Optional[str], str]] = None
|
|
|
|
def __str__(self) -> str:
|
|
return " | ".join(str(alt) for alt in self.alts)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Rhs({self.alts!r})"
|
|
|
|
def __iter__(self) -> Iterator[List[Alt]]:
|
|
yield self.alts
|
|
|
|
@property
|
|
def can_be_inlined(self) -> bool:
|
|
if len(self.alts) != 1 or len(self.alts[0].items) != 1:
|
|
return False
|
|
# If the alternative has an action we cannot inline
|
|
if getattr(self.alts[0], "action", None) is not None:
|
|
return False
|
|
return True
|
|
|
|
|
|
class Alt:
|
|
def __init__(self, items: List[NamedItem], *, icut: int = -1, action: Optional[str] = None):
|
|
self.items = items
|
|
self.icut = icut
|
|
self.action = action
|
|
|
|
def __str__(self) -> str:
|
|
core = " ".join(str(item) for item in self.items)
|
|
if not SIMPLE_STR and self.action:
|
|
return f"{core} {{ {self.action} }}"
|
|
else:
|
|
return core
|
|
|
|
def __repr__(self) -> str:
|
|
args = [repr(self.items)]
|
|
if self.icut >= 0:
|
|
args.append(f"icut={self.icut}")
|
|
if self.action:
|
|
args.append(f"action={self.action!r}")
|
|
return f"Alt({', '.join(args)})"
|
|
|
|
def __iter__(self) -> Iterator[List[NamedItem]]:
|
|
yield self.items
|
|
|
|
|
|
class NamedItem:
|
|
def __init__(self, name: Optional[str], item: Item, type: Optional[str] = None):
|
|
self.name = name
|
|
self.item = item
|
|
self.type = type
|
|
|
|
def __str__(self) -> str:
|
|
if not SIMPLE_STR and self.name:
|
|
return f"{self.name}={self.item}"
|
|
else:
|
|
return str(self.item)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"NamedItem({self.name!r}, {self.item!r})"
|
|
|
|
def __iter__(self) -> Iterator[Item]:
|
|
yield self.item
|
|
|
|
|
|
class Forced:
|
|
def __init__(self, node: Plain):
|
|
self.node = node
|
|
|
|
def __str__(self) -> str:
|
|
return f"&&{self.node}"
|
|
|
|
def __iter__(self) -> Iterator[Plain]:
|
|
yield self.node
|
|
|
|
|
|
class Lookahead:
|
|
def __init__(self, node: Plain, sign: str):
|
|
self.node = node
|
|
self.sign = sign
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.sign}{self.node}"
|
|
|
|
def __iter__(self) -> Iterator[Plain]:
|
|
yield self.node
|
|
|
|
|
|
class PositiveLookahead(Lookahead):
|
|
def __init__(self, node: Plain):
|
|
super().__init__(node, "&")
|
|
|
|
def __repr__(self) -> str:
|
|
return f"PositiveLookahead({self.node!r})"
|
|
|
|
|
|
class NegativeLookahead(Lookahead):
|
|
def __init__(self, node: Plain):
|
|
super().__init__(node, "!")
|
|
|
|
def __repr__(self) -> str:
|
|
return f"NegativeLookahead({self.node!r})"
|
|
|
|
|
|
class Opt:
|
|
def __init__(self, node: Item):
|
|
self.node = node
|
|
|
|
def __str__(self) -> str:
|
|
s = str(self.node)
|
|
# TODO: Decide whether to use [X] or X? based on type of X
|
|
if " " in s:
|
|
return f"[{s}]"
|
|
else:
|
|
return f"{s}?"
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Opt({self.node!r})"
|
|
|
|
def __iter__(self) -> Iterator[Item]:
|
|
yield self.node
|
|
|
|
|
|
class Repeat:
|
|
"""Shared base class for x* and x+."""
|
|
|
|
def __init__(self, node: Plain):
|
|
self.node = node
|
|
self.memo: Optional[Tuple[Optional[str], str]] = None
|
|
|
|
def __iter__(self) -> Iterator[Plain]:
|
|
yield self.node
|
|
|
|
|
|
class Repeat0(Repeat):
|
|
def __str__(self) -> str:
|
|
s = str(self.node)
|
|
# TODO: Decide whether to use (X)* or X* based on type of X
|
|
if " " in s:
|
|
return f"({s})*"
|
|
else:
|
|
return f"{s}*"
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Repeat0({self.node!r})"
|
|
|
|
|
|
class Repeat1(Repeat):
|
|
def __str__(self) -> str:
|
|
s = str(self.node)
|
|
# TODO: Decide whether to use (X)+ or X+ based on type of X
|
|
if " " in s:
|
|
return f"({s})+"
|
|
else:
|
|
return f"{s}+"
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Repeat1({self.node!r})"
|
|
|
|
|
|
class Gather(Repeat):
|
|
def __init__(self, separator: Plain, node: Plain):
|
|
self.separator = separator
|
|
self.node = node
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.separator!s}.{self.node!s}+"
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Gather({self.separator!r}, {self.node!r})"
|
|
|
|
|
|
class Group:
|
|
def __init__(self, rhs: Rhs):
|
|
self.rhs = rhs
|
|
|
|
def __str__(self) -> str:
|
|
return f"({self.rhs})"
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Group({self.rhs!r})"
|
|
|
|
def __iter__(self) -> Iterator[Rhs]:
|
|
yield self.rhs
|
|
|
|
|
|
class Cut:
|
|
def __init__(self) -> None:
|
|
pass
|
|
|
|
def __repr__(self) -> str:
|
|
return f"Cut()"
|
|
|
|
def __str__(self) -> str:
|
|
return f"~"
|
|
|
|
def __iter__(self) -> Iterator[Tuple[str, str]]:
|
|
if False:
|
|
yield
|
|
|
|
def __eq__(self, other: object) -> bool:
|
|
if not isinstance(other, Cut):
|
|
return NotImplemented
|
|
return True
|
|
|
|
def initial_names(self) -> AbstractSet[str]:
|
|
return set()
|
|
|
|
|
|
Plain = Union[Leaf, Group]
|
|
Item = Union[Plain, Opt, Repeat, Forced, Lookahead, Rhs, Cut]
|
|
RuleName = Tuple[str, str]
|
|
MetaTuple = Tuple[str, Optional[str]]
|
|
MetaList = List[MetaTuple]
|
|
RuleList = List[Rule]
|
|
NamedItemList = List[NamedItem]
|
|
LookaheadOrCut = Union[Lookahead, Cut]
|