mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-27 02:39:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

gh-125588: Allow to regenerate the parser with Python < 3.12 (#127969)

Browse source 
Signed-off-by: Pablo Galindo <pablogsal@gmail.com>
This commit is contained in:
Pablo Galindo Salgado 2024-12-15 23:17:01 +00:00 • committed by GitHub
parent b74c8f58e8
commit 47c5a0f307
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Tools/peg_generator/pegen/parser.py
View file

@ -207,7 +207,7 @@ class Parser:
@memoize @memoize
def fstring_start(self) -> Optional[tokenize.TokenInfo]: def fstring_start(self) -> Optional[tokenize.TokenInfo]:
FSTRING_START = getattr(token, "FSTRING_START") FSTRING_START = getattr(token, "FSTRING_START", None)
if not FSTRING_START: if not FSTRING_START:
return None return None
tok = self._tokenizer.peek() tok = self._tokenizer.peek()
@ -217,7 +217,7 @@ class Parser:
@memoize @memoize
def fstring_middle(self) -> Optional[tokenize.TokenInfo]: def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None)
if not FSTRING_MIDDLE: if not FSTRING_MIDDLE:
return None return None
tok = self._tokenizer.peek() tok = self._tokenizer.peek()
@ -227,7 +227,7 @@ class Parser:
@memoize @memoize
def fstring_end(self) -> Optional[tokenize.TokenInfo]: def fstring_end(self) -> Optional[tokenize.TokenInfo]:
FSTRING_END = getattr(token, "FSTRING_END") FSTRING_END = getattr(token, "FSTRING_END", None)
if not FSTRING_END: if not FSTRING_END:
return None return None
tok = self._tokenizer.peek() tok = self._tokenizer.peek()

6
Tools/peg_generator/pegen/parser_generator.py
View file

@ -1,3 +1,4 @@
import sys
import ast import ast
import contextlib import contextlib
import re import re
@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor):
def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): def __init__(self, rules: Dict[str, Rule], tokens: Set[str]):
self.rules = rules self.rules = rules
self.tokens = tokens self.tokens = tokens
# If python < 3.12 add the virtual fstring tokens
if sys.version_info < (3, 12):
self.tokens.add("FSTRING_START")
self.tokens.add("FSTRING_END")
self.tokens.add("FSTRING_MIDDLE")
def visit_NameLeaf(self, node: NameLeaf) -> None: def visit_NameLeaf(self, node: NameLeaf) -> None:
if node.value not in self.rules and node.value not in self.tokens: if node.value not in self.rules and node.value not in self.tokens: