mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 14:07:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

bpo-40334: Refactor peg_generator to receive a Tokens file when building c code (GH-19745)

Browse source
This commit is contained in:
Pablo Galindo 2020-04-28 13:11:55 +01:00 • committed by GitHub
parent 3d53d8756f
commit 5b9f4988c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 220 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Tools/peg_generator/pegen/testutil.py
View file

@ -5,6 +5,7 @@ import pathlib
import sys
import textwrap
import tokenize
import token
from typing import Any, cast, Dict, IO, Type, Final
@ -16,6 +17,11 @@ from pegen.parser import Parser
from pegen.python_generator import PythonParserGenerator
from pegen.tokenizer import Tokenizer
EXACT_TOKENS = token.EXACT_TOKEN_TYPES # type: ignore
NON_EXACT_TOKENS = {
name for index, name in token.tok_name.items() if index not in EXACT_TOKENS.values()
}
def generate_parser(grammar: Grammar) -> Type[Parser]:
# Generate a parser.
@ -70,7 +76,7 @@ def import_file(full_name: str, path: str) -> Any:
def generate_c_parser_source(grammar: Grammar) -> str:
out = io.StringIO()
genr = CParserGenerator(grammar, out)
genr = CParserGenerator(grammar, EXACT_TOKENS, NON_EXACT_TOKENS, out)
genr.generate("<string>")
return out.getvalue()
@ -90,7 +96,7 @@ def generate_parser_c_extension(
assert not os.listdir(path)
source = path / "parse.c"
with open(source, "w") as file:
genr = CParserGenerator(grammar, file, debug=debug)
genr = CParserGenerator(grammar, EXACT_TOKENS, NON_EXACT_TOKENS, file, debug=debug)
genr.generate("parse.c")
compile_c_extension(str(source), build_dir=str(path))