mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-22 16:33:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-40688: Use the correct parser in the peg_generator scripts (GH-20235)

Browse source 
The scripts in `Tools/peg_generator/scripts` mostly assume that
`ast.parse` and `compile` use the old parser, since this was the
state of things, while we were developing them. They need to be
updated to always use the correct parser. `_peg_parser` is being
extended to support both parsing and compiling with both parsers.
(cherry picked from commit 9645930b5b)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Miss Islington (bot) 2020-05-25 13:11:36 -07:00 • committed by GitHub
parent 318a18eb88
commit 3c6c86ab77
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 151 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

41
Tools/peg_generator/scripts/benchmark.py
View file

@ -6,6 +6,8 @@ import sys
import os
from time import time
import _peg_parser
try:
import memory_profiler
except ModuleNotFoundError:
@ -14,8 +16,6 @@ except ModuleNotFoundError:
sys.exit(1)
sys.path.insert(0, os.getcwd())
from peg_extension import parse
from pegen.build import build_c_parser_and_generator
from scripts.test_parse_directory import parse_directory
argparser = argparse.ArgumentParser(
@ -41,9 +41,6 @@ command_compile = subcommands.add_parser(
"compile", help="Benchmark parsing and compiling to bytecode"
)
command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST")
command_check = subcommands.add_parser(
"check", help="Benchmark parsing and throwing the tree away"
)
def benchmark(func):
@ -66,22 +63,20 @@ def benchmark(func):
@benchmark
def time_compile(source, parser):
if parser == "cpython":
return compile(source, os.path.join("data", "xxl.py"), "exec")
return _peg_parser.compile_string(
source,
oldparser=True,
)
else:
return parse.parse_string(source, mode=2)
return _peg_parser.compile_string(source)
@benchmark
def time_parse(source, parser):
if parser == "cpython":
return ast.parse(source, os.path.join("data", "xxl.py"), "exec")
return _peg_parser.parse_string(source, oldparser=True)
else:
return parse.parse_string(source, mode=1)
@benchmark
def time_check(source):
return parse.parse_string(source, mode=0)
return _peg_parser.parse_string(source)
def run_benchmark_xxl(subcommand, parser, source):
@ -89,32 +84,20 @@ def run_benchmark_xxl(subcommand, parser, source):
time_compile(source, parser)
elif subcommand == "parse":
time_parse(source, parser)
elif subcommand == "check":
time_check(source)
def run_benchmark_stdlib(subcommand, parser):
modes = {"compile": 2, "parse": 1, "check": 0}
extension = None
if parser == "pegen":
extension = build_c_parser_and_generator(
"../../Grammar/python.gram",
"../../Grammar/Tokens",
"peg_extension/parse.c",
compile_extension=True,
skip_actions=False,
)
for _ in range(3):
parse_directory(
"../../Lib",
"../../Grammar/python.gram",
"../../Grammar/Tokens",
verbose=False,
excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
skip_actions=False,
tree_arg=0,
short=True,
extension=extension,
mode=modes[subcommand],
mode=2 if subcommand == "compile" else 1,
parser=parser,
)
@ -127,8 +110,6 @@ def main():
if subcommand is None:
argparser.error("A benchmark to run is required")
if subcommand == "check" and parser == "cpython":
argparser.error("Cannot use check target with the CPython parser")
if target == "xxl":
with open(os.path.join("data", "xxl.py"), "r") as f: