mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-40688: Use the correct parser in the peg_generator scripts (GH-20235)

Browse source 
The scripts in `Tools/peg_generator/scripts` mostly assume that
`ast.parse` and `compile` use the old parser, since this was the
state of things, while we were developing them. They need to be
updated to always use the correct parser. `_peg_parser` is being
extended to support both parsing and compiling with both parsers.
This commit is contained in:
Lysandros Nikolaou 2020-05-25 22:51:58 +03:00 • committed by GitHub
parent 448325369f
commit 9645930b5b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 151 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

163
Modules/_peg_parser.c
View file

@ -1,60 +1,9 @@
#include <Python.h>
#include "pegen_interface.h"
PyObject *
_Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds)
static int
_mode_str_to_int(char *mode_str)
{
static char *keywords[] = {"file", "mode", NULL};
char *filename;
char *mode_str = "exec";
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", keywords, &filename, &mode_str)) {
return NULL;
}
int mode;
if (strcmp(mode_str, "exec") == 0) {
mode = Py_file_input;
}
else if (strcmp(mode_str, "single") == 0) {
mode = Py_single_input;
}
else {
return PyErr_Format(PyExc_ValueError, "mode must be either 'exec' or 'single'");
}
PyArena *arena = PyArena_New();
if (arena == NULL) {
return NULL;
}
PyCompilerFlags flags = _PyCompilerFlags_INIT;
PyObject *result = NULL;
mod_ty res = PyPegen_ASTFromFilename(filename, mode, &flags, arena);
if (res == NULL) {
goto error;
}
result = PyAST_mod2obj(res);
error:
PyArena_Free(arena);
return result;
}
PyObject *
_Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *keywords[] = {"string", "mode", "oldparser", NULL};
char *the_string;
char *mode_str = "exec";
int oldparser = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sp", keywords,
&the_string, &mode_str, &oldparser)) {
return NULL;
}
int mode;
if (strcmp(mode_str, "exec") == 0) {
mode = Py_file_input;
@ -66,39 +15,119 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
mode = Py_single_input;
}
else {
mode = -1;
}
return mode;
}
static mod_ty
_run_parser(char *str, char *filename, int mode, PyCompilerFlags *flags, PyArena *arena, int oldparser)
{
mod_ty mod;
if (!oldparser) {
mod = PyPegen_ASTFromString(str, filename, mode, flags, arena);
}
else {
mod = PyParser_ASTFromString(str, filename, mode, flags, arena);
}
return mod;
}
PyObject *
_Py_compile_string(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL};
char *the_string;
char *filename = "<string>";
char *mode_str = "exec";
int oldparser = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords,
&the_string, &filename, &mode_str, &oldparser)) {
return NULL;
}
int mode = _mode_str_to_int(mode_str);
if (mode == -1) {
return PyErr_Format(PyExc_ValueError, "mode must be either 'exec' or 'eval' or 'single'");
}
PyCompilerFlags flags = _PyCompilerFlags_INIT;
flags.cf_flags = PyCF_IGNORE_COOKIE;
PyArena *arena = PyArena_New();
if (arena == NULL) {
return NULL;
}
PyObject *result = NULL;
mod_ty mod = _run_parser(the_string, filename, mode, &flags, arena, oldparser);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
}
PyObject *filename_ob = PyUnicode_DecodeFSDefault(filename);
if (filename_ob == NULL) {
PyArena_Free(arena);
return NULL;
}
PyCodeObject *result = PyAST_CompileObject(mod, filename_ob, &flags, -1, arena);
Py_XDECREF(filename_ob);
PyArena_Free(arena);
return (PyObject *)result;
}
PyObject *
_Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL};
char *the_string;
char *filename = "<string>";
char *mode_str = "exec";
int oldparser = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords,
&the_string, &filename, &mode_str, &oldparser)) {
return NULL;
}
int mode = _mode_str_to_int(mode_str);
if (mode == -1) {
return PyErr_Format(PyExc_ValueError, "mode must be either 'exec' or 'eval' or 'single'");
}
PyCompilerFlags flags = _PyCompilerFlags_INIT;
flags.cf_flags = PyCF_IGNORE_COOKIE;
mod_ty res;
if (oldparser) {
res = PyParser_ASTFromString(the_string, "<string>", mode, &flags, arena);
PyArena *arena = PyArena_New();
if (arena == NULL) {
return NULL;
}
else {
res = PyPegen_ASTFromString(the_string, "<string>", mode, &flags, arena);
}
if (res == NULL) {
goto error;
}
result = PyAST_mod2obj(res);
error:
mod_ty mod = _run_parser(the_string, filename, mode, &flags, arena, oldparser);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
}
PyObject *result = PyAST_mod2obj(mod);
PyArena_Free(arena);
return result;
}
static PyMethodDef ParseMethods[] = {
{"parse_file", (PyCFunction)(void (*)(void))_Py_parse_file, METH_VARARGS|METH_KEYWORDS, "Parse a file."},
{"parse_string", (PyCFunction)(void (*)(void))_Py_parse_string, METH_VARARGS|METH_KEYWORDS,"Parse a string."},
{
"parse_string",
(PyCFunction)(void (*)(void))_Py_parse_string,
METH_VARARGS|METH_KEYWORDS,
"Parse a string, return an AST."
},
{
"compile_string",
(PyCFunction)(void (*)(void))_Py_compile_string,
METH_VARARGS|METH_KEYWORDS,
"Compile a string, return a code object."
},
{NULL, NULL, 0, NULL} /* Sentinel */
};