From b48599b8017f896c965e89c402bb70feef49808a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Thu, 8 May 2025 03:28:20 +0200 Subject: [PATCH] gh-133555: Allow regenerating the parser with Python < 3.14 (#133557) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Grammar/Tokens | 5 +++++ Tools/peg_generator/pegen/parser_generator.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Grammar/Tokens b/Grammar/Tokens index e40a4437afb..0547e6ed08f 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,3 +1,8 @@ +# When adding new tokens, remember to update the PEG generator in +# Tools/peg_generator/pegen/parser_generator.py +# This will ensure that older versions of Python can generate a Python parser +# using "python -m pegen python ". + ENDMARKER NAME NUMBER diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 6ce0649aefe..52ae743c26b 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -81,6 +81,11 @@ class RuleCheckingVisitor(GrammarVisitor): self.tokens.add("FSTRING_START") self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") + # If python < 3.14 add the virtual tstring tokens + if sys.version_info < (3, 14, 0, 'beta', 1): + self.tokens.add("TSTRING_START") + self.tokens.add("TSTRING_END") + self.tokens.add("TSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: