mirror of
https://github.com/python/cpython.git
synced 2025-08-02 16:13:13 +00:00
bpo-40688: Use the correct parser in the peg_generator scripts (GH-20235)
The scripts in `Tools/peg_generator/scripts` mostly assume that `ast.parse` and `compile` use the old parser, since this was the state of things, while we were developing them. They need to be updated to always use the correct parser. `_peg_parser` is being extended to support both parsing and compiling with both parsers.
This commit is contained in:
Lysandros Nikolaou
GitHub
parent
448325369f
commit
9645930b5b
6 changed files with 151 additions and 187 deletions
|
|
@ -6,6 +6,8 @@ import sys
|
|||
import os
|
||||
from time import time
|
||||
|
||||
import _peg_parser
|
||||
|
||||
try:
|
||||
import memory_profiler
|
||||
except ModuleNotFoundError:
|
||||
|
|
@ -14,8 +16,6 @@ except ModuleNotFoundError:
|
|||
sys.exit(1)
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
from peg_extension import parse
|
||||
from pegen.build import build_c_parser_and_generator
|
||||
from scripts.test_parse_directory import parse_directory
|
||||
|
||||
argparser = argparse.ArgumentParser(
|
||||
|
|
@ -41,9 +41,6 @@ command_compile = subcommands.add_parser(
|
|||
"compile", help="Benchmark parsing and compiling to bytecode"
|
||||
)
|
||||
command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST")
|
||||
command_check = subcommands.add_parser(
|
||||
"check", help="Benchmark parsing and throwing the tree away"
|
||||
)
|
||||
|
||||
|
||||
def benchmark(func):
|
||||
|
|
@ -66,22 +63,20 @@ def benchmark(func):
|
|||
@benchmark
|
||||
def time_compile(source, parser):
|
||||
if parser == "cpython":
|
||||
return compile(source, os.path.join("data", "xxl.py"), "exec")
|
||||
return _peg_parser.compile_string(
|
||||
source,
|
||||
oldparser=True,
|
||||
)
|
||||
else:
|
||||
return parse.parse_string(source, mode=2)
|
||||
return _peg_parser.compile_string(source)
|
||||
|
||||
|
||||
@benchmark
|
||||
def time_parse(source, parser):
|
||||
if parser == "cpython":
|
||||
return ast.parse(source, os.path.join("data", "xxl.py"), "exec")
|
||||
return _peg_parser.parse_string(source, oldparser=True)
|
||||
else:
|
||||
return parse.parse_string(source, mode=1)
|
||||
|
||||
|
||||
@benchmark
|
||||
def time_check(source):
|
||||
return parse.parse_string(source, mode=0)
|
||||
return _peg_parser.parse_string(source)
|
||||
|
||||
|
||||
def run_benchmark_xxl(subcommand, parser, source):
|
||||
|
|
@ -89,32 +84,20 @@ def run_benchmark_xxl(subcommand, parser, source):
|
|||
time_compile(source, parser)
|
||||
elif subcommand == "parse":
|
||||
time_parse(source, parser)
|
||||
elif subcommand == "check":
|
||||
time_check(source)
|
||||
|
||||
|
||||
def run_benchmark_stdlib(subcommand, parser):
|
||||
modes = {"compile": 2, "parse": 1, "check": 0}
|
||||
extension = None
|
||||
if parser == "pegen":
|
||||
extension = build_c_parser_and_generator(
|
||||
"../../Grammar/python.gram",
|
||||
"../../Grammar/Tokens",
|
||||
"peg_extension/parse.c",
|
||||
compile_extension=True,
|
||||
skip_actions=False,
|
||||
)
|
||||
for _ in range(3):
|
||||
parse_directory(
|
||||
"../../Lib",
|
||||
"../../Grammar/python.gram",
|
||||
"../../Grammar/Tokens",
|
||||
verbose=False,
|
||||
excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
|
||||
skip_actions=False,
|
||||
tree_arg=0,
|
||||
short=True,
|
||||
extension=extension,
|
||||
mode=modes[subcommand],
|
||||
mode=2 if subcommand == "compile" else 1,
|
||||
parser=parser,
|
||||
)
|
||||
|
||||
|
|
@ -127,8 +110,6 @@ def main():
|
|||
|
||||
if subcommand is None:
|
||||
argparser.error("A benchmark to run is required")
|
||||
if subcommand == "check" and parser == "cpython":
|
||||
argparser.error("Cannot use check target with the CPython parser")
|
||||
|
||||
if target == "xxl":
|
||||
with open(os.path.join("data", "xxl.py"), "r") as f:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ import os
|
|||
import sys
|
||||
import tempfile
|
||||
|
||||
import _peg_parser
|
||||
|
||||
from typing import List
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
|
|
@ -72,7 +74,7 @@ def diff_trees(a: ast.AST, b: ast.AST, verbose: bool = False) -> List[str]:
|
|||
|
||||
|
||||
def show_parse(source: str, verbose: bool = False) -> str:
|
||||
tree = ast.parse(source)
|
||||
tree = _peg_parser.parse_string(source, oldparser=True)
|
||||
return format_tree(tree, verbose).rstrip("\n")
|
||||
|
||||
|
||||
|
|
@ -90,17 +92,11 @@ def main() -> None:
|
|||
sep = " "
|
||||
program = sep.join(args.program)
|
||||
if args.grammar_file:
|
||||
sys.path.insert(0, os.curdir)
|
||||
from pegen.build import build_parser_and_generator
|
||||
|
||||
build_parser_and_generator(args.grammar_file, "peg_parser/parse.c", compile_extension=True)
|
||||
from pegen.parse import parse_string # type: ignore[import]
|
||||
|
||||
tree = parse_string(program, mode=1)
|
||||
tree = _peg_parser.parse_string(program)
|
||||
|
||||
if args.diff:
|
||||
a = tree
|
||||
b = ast.parse(program)
|
||||
b = _peg_parser.parse_string(program, oldparser=True)
|
||||
diff = diff_trees(a, b, args.verbose)
|
||||
if diff:
|
||||
for line in diff:
|
||||
|
|
@ -111,8 +107,8 @@ def main() -> None:
|
|||
print(f"# Parsed using {args.grammar_file}")
|
||||
print(format_tree(tree, args.verbose))
|
||||
else:
|
||||
tree = ast.parse(program)
|
||||
print("# Parse using ast.parse()")
|
||||
tree = _peg_parser.parse_string(program, oldparser=True)
|
||||
print("# Parse using the old parser")
|
||||
print(format_tree(tree, args.verbose))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,14 @@ import os
|
|||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import tokenize
|
||||
import _peg_parser
|
||||
from glob import glob
|
||||
from pathlib import PurePath
|
||||
|
||||
from typing import List, Optional, Any
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
from pegen.build import build_c_parser_and_generator
|
||||
from pegen.ast_dump import ast_dump
|
||||
from pegen.testutil import print_memstats
|
||||
from scripts import show_parse
|
||||
|
|
@ -83,7 +84,7 @@ def compare_trees(
|
|||
actual_tree: ast.AST, file: str, verbose: bool, include_attributes: bool = False,
|
||||
) -> int:
|
||||
with open(file) as f:
|
||||
expected_tree = ast.parse(f.read())
|
||||
expected_tree = _peg_parser.parse_string(f.read(), oldparser=True)
|
||||
|
||||
expected_text = ast_dump(expected_tree, include_attributes=include_attributes)
|
||||
actual_text = ast_dump(actual_tree, include_attributes=include_attributes)
|
||||
|
|
@ -121,7 +122,6 @@ def parse_directory(
|
|||
skip_actions: bool,
|
||||
tree_arg: int,
|
||||
short: bool,
|
||||
extension: Any,
|
||||
mode: int,
|
||||
parser: str,
|
||||
) -> int:
|
||||
|
|
@ -137,47 +137,21 @@ def parse_directory(
|
|||
if not os.path.exists(grammar_file):
|
||||
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
if not extension and parser == "pegen":
|
||||
build_c_parser_and_generator(
|
||||
grammar_file,
|
||||
tokens_file,
|
||||
"peg_extension/parse.c",
|
||||
compile_extension=True,
|
||||
skip_actions=skip_actions,
|
||||
)
|
||||
except Exception as err:
|
||||
print(
|
||||
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
traceback.print_exception(err.__class__, err, None)
|
||||
|
||||
return 1
|
||||
|
||||
else:
|
||||
print(
|
||||
"A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n"
|
||||
)
|
||||
|
||||
if parser == "pegen":
|
||||
try:
|
||||
from peg_extension import parse # type: ignore
|
||||
except Exception as e:
|
||||
print(
|
||||
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if tree_arg:
|
||||
assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees"
|
||||
|
||||
# For a given directory, traverse files and attempt to parse each one
|
||||
# - Output success/failure for each file
|
||||
errors = 0
|
||||
files = []
|
||||
trees = {} # Trees to compare (after everything else is done)
|
||||
total_seconds = 0
|
||||
|
||||
t0 = time.time()
|
||||
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
|
||||
# Only attempt to parse Python files and files that are not excluded
|
||||
should_exclude_file = False
|
||||
|
|
@ -187,25 +161,31 @@ def parse_directory(
|
|||
break
|
||||
|
||||
if not should_exclude_file:
|
||||
with tokenize.open(file) as f:
|
||||
source = f.read()
|
||||
try:
|
||||
if tree_arg:
|
||||
mode = 1
|
||||
if parser == "cpython":
|
||||
with open(file, "r") as f:
|
||||
source = f.read()
|
||||
if mode == 2:
|
||||
compile(source, file, "exec")
|
||||
elif mode == 1:
|
||||
ast.parse(source, file, "exec")
|
||||
t0 = time.time()
|
||||
if mode == 2:
|
||||
result = _peg_parser.compile_string(
|
||||
source,
|
||||
filename=file,
|
||||
oldparser=parser == "cpython",
|
||||
)
|
||||
else:
|
||||
tree = parse.parse_file(file, mode=mode)
|
||||
result = _peg_parser.parse_string(
|
||||
source,
|
||||
filename=file,
|
||||
oldparser=parser == "cpython"
|
||||
)
|
||||
t1 = time.time()
|
||||
total_seconds += (t1 - t0)
|
||||
if tree_arg:
|
||||
trees[file] = tree
|
||||
trees[file] = result
|
||||
if not short:
|
||||
report_status(succeeded=True, file=file, verbose=verbose)
|
||||
except Exception as error:
|
||||
try:
|
||||
ast.parse(file)
|
||||
_peg_parser.parse_string(source, mode="exec", oldparser=True)
|
||||
except Exception:
|
||||
if not short:
|
||||
print(f"File {file} cannot be parsed by either pegen or the ast module.")
|
||||
|
|
@ -217,7 +197,6 @@ def parse_directory(
|
|||
files.append(file)
|
||||
t1 = time.time()
|
||||
|
||||
total_seconds = t1 - t0
|
||||
total_files = len(files)
|
||||
|
||||
total_bytes = 0
|
||||
|
|
@ -238,13 +217,6 @@ def parse_directory(
|
|||
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
|
||||
)
|
||||
|
||||
if parser == "pegen":
|
||||
# Dump memo stats to @data.
|
||||
with open("@data", "w") as datafile:
|
||||
for i, count in enumerate(parse.get_memo_stats()):
|
||||
if count:
|
||||
datafile.write(f"{i:4d} {count:9d}\n")
|
||||
|
||||
if short:
|
||||
print_memstats()
|
||||
|
||||
|
|
@ -275,6 +247,7 @@ def main() -> None:
|
|||
skip_actions = args.skip_actions
|
||||
tree = args.tree
|
||||
short = args.short
|
||||
mode = 1 if args.tree else 2
|
||||
sys.exit(
|
||||
parse_directory(
|
||||
directory,
|
||||
|
|
@ -285,8 +258,7 @@ def main() -> None:
|
|||
skip_actions,
|
||||
tree,
|
||||
short,
|
||||
None,
|
||||
0,
|
||||
mode,
|
||||
"pegen",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def find_dirname(package_name: str) -> str:
|
|||
assert False # This is to fix mypy, should never be reached
|
||||
|
||||
|
||||
def run_tests(dirname: str, tree: int, extension: Any) -> int:
|
||||
def run_tests(dirname: str, tree: int) -> int:
|
||||
return test_parse_directory.parse_directory(
|
||||
dirname,
|
||||
HERE / ".." / ".." / ".." / "Grammar" / "python.gram",
|
||||
|
|
@ -72,7 +72,6 @@ def run_tests(dirname: str, tree: int, extension: Any) -> int:
|
|||
skip_actions=False,
|
||||
tree_arg=tree,
|
||||
short=True,
|
||||
extension=extension,
|
||||
mode=1,
|
||||
parser="pegen",
|
||||
)
|
||||
|
|
@ -82,13 +81,6 @@ def main() -> None:
|
|||
args = argparser.parse_args()
|
||||
tree = args.tree
|
||||
|
||||
extension = build.build_c_parser_and_generator(
|
||||
HERE / ".." / ".." / ".." / "Grammar" / "python.gram",
|
||||
HERE / ".." / ".." / ".." / "Grammar" / "Tokens",
|
||||
"peg_extension/parse.c",
|
||||
compile_extension=True,
|
||||
)
|
||||
|
||||
for package in get_packages():
|
||||
print(f"Extracting files from {package}... ", end="")
|
||||
try:
|
||||
|
|
@ -100,7 +92,7 @@ def main() -> None:
|
|||
|
||||
print(f"Trying to parse all python files ... ")
|
||||
dirname = find_dirname(package)
|
||||
status = run_tests(dirname, tree, extension)
|
||||
status = run_tests(dirname, tree)
|
||||
if status == 0:
|
||||
shutil.rmtree(dirname)
|
||||
else:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue