bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503)

Co-authored-by: Guido van Rossum <guido@python.org>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Pablo Galindo 2020-04-22 23:29:27 +01:00 committed by GitHub
parent a81849b031
commit c5fc156852
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
91 changed files with 27057 additions and 146 deletions

View file

@ -563,7 +563,8 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
CALL(fold_tuple, expr_ty, node_);
break;
case Name_kind:
if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
if (node_->v.Name.ctx == Load &&
_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
return make_const(node_, PyBool_FromLong(!state->optimize), ctx_);
}
break;

View file

@ -816,7 +816,12 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
if (str == NULL)
goto error;
int current_use_peg = PyInterpreterState_Get()->config.use_peg;
if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) {
PyInterpreterState_Get()->config.use_peg = 0;
}
result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
PyInterpreterState_Get()->config.use_peg = current_use_peg;
Py_XDECREF(source_copy);
goto finally;

View file

@ -2152,6 +2152,55 @@ compiler_default_arguments(struct compiler *c, arguments_ty args)
return funcflags;
}
static int
forbidden_name(struct compiler *c, identifier name, expr_context_ty ctx)
{
if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) {
compiler_error(c, "cannot assign to __debug__");
return 1;
}
return 0;
}
static int
compiler_check_debug_one_arg(struct compiler *c, arg_ty arg)
{
if (arg != NULL) {
if (forbidden_name(c, arg->arg, Store))
return 0;
}
return 1;
}
static int
compiler_check_debug_args_seq(struct compiler *c, asdl_seq *args)
{
if (args != NULL) {
for (int i = 0, n = asdl_seq_LEN(args); i < n; i++) {
if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i)))
return 0;
}
}
return 1;
}
static int
compiler_check_debug_args(struct compiler *c, arguments_ty args)
{
if (!compiler_check_debug_args_seq(c, args->posonlyargs))
return 0;
if (!compiler_check_debug_args_seq(c, args->args))
return 0;
if (!compiler_check_debug_one_arg(c, args->vararg))
return 0;
if (!compiler_check_debug_args_seq(c, args->kwonlyargs))
return 0;
if (!compiler_check_debug_one_arg(c, args->kwarg))
return 0;
return 1;
}
static int
compiler_function(struct compiler *c, stmt_ty s, int is_async)
{
@ -2189,6 +2238,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
scope_type = COMPILER_SCOPE_FUNCTION;
}
if (!compiler_check_debug_args(c, args))
return 0;
if (!compiler_decorators(c, decos))
return 0;
@ -2596,6 +2648,9 @@ compiler_lambda(struct compiler *c, expr_ty e)
arguments_ty args = e->v.Lambda.args;
assert(e->kind == Lambda_kind);
if (!compiler_check_debug_args(c, args))
return 0;
if (!name) {
name = PyUnicode_InternFromString("<lambda>");
if (!name)
@ -3505,6 +3560,9 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
!_PyUnicode_EqualToASCIIString(name, "True") &&
!_PyUnicode_EqualToASCIIString(name, "False"));
if (forbidden_name(c, name, ctx))
return 0;
mangled = _Py_Mangle(c->u->u_private, name);
if (!mangled)
return 0;
@ -4056,6 +4114,9 @@ validate_keywords(struct compiler *c, asdl_seq *keywords)
if (key->arg == NULL) {
continue;
}
if (forbidden_name(c, key->arg, Store)) {
return -1;
}
for (Py_ssize_t j = i + 1; j < nkeywords; j++) {
keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j));
if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) {
@ -5013,6 +5074,8 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names);
break;
case Store:
if (forbidden_name(c, e->v.Attribute.attr, e->v.Attribute.ctx))
return 0;
ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names);
break;
case Del:
@ -5183,6 +5246,8 @@ compiler_annassign(struct compiler *c, stmt_ty s)
}
switch (targ->kind) {
case Name_kind:
if (forbidden_name(c, targ->v.Name.id, Store))
return 0;
/* If we have a simple name in a module or class, store annotation. */
if (s->v.AnnAssign.simple &&
(c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
@ -5200,6 +5265,8 @@ compiler_annassign(struct compiler *c, stmt_ty s)
}
break;
case Attribute_kind:
if (forbidden_name(c, targ->v.Attribute.attr, Store))
return 0;
if (!s->v.AnnAssign.value &&
!check_ann_expr(c, targ->v.Attribute.value)) {
return 0;

89
Python/importlib.h generated
View file

@ -1594,50 +1594,51 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
0,218,1,120,90,5,119,104,101,114,101,90,9,102,114,111,
109,95,110,97,109,101,90,3,101,120,99,114,10,0,0,0,
114,10,0,0,0,114,11,0,0,0,114,215,0,0,0,9,
4,0,0,115,44,0,0,0,0,10,8,1,10,1,4,1,
12,2,4,1,28,2,8,1,14,1,10,1,2,255,8,2,
10,1,14,1,2,1,14,1,14,4,10,1,16,255,2,2,
12,1,26,1,114,215,0,0,0,99,1,0,0,0,0,0,
0,0,0,0,0,0,3,0,0,0,6,0,0,0,67,0,
0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,125,
1,124,0,160,0,100,2,161,1,125,2,124,1,100,3,117,
1,114,82,124,2,100,3,117,1,114,78,124,1,124,2,106,
1,107,3,114,78,116,2,106,3,100,4,124,1,155,2,100,
5,124,2,106,1,155,2,100,6,157,5,116,4,100,7,100,
8,141,3,1,0,124,1,83,0,124,2,100,3,117,1,114,
96,124,2,106,1,83,0,116,2,106,3,100,9,116,4,100,
7,100,8,141,3,1,0,124,0,100,10,25,0,125,1,100,
11,124,0,118,1,114,142,124,1,160,5,100,12,161,1,100,
13,25,0,125,1,124,1,83,0,41,14,122,167,67,97,108,
99,117,108,97,116,101,32,119,104,97,116,32,95,95,112,97,
99,107,97,103,101,95,95,32,115,104,111,117,108,100,32,98,
101,46,10,10,32,32,32,32,95,95,112,97,99,107,97,103,
101,95,95,32,105,115,32,110,111,116,32,103,117,97,114,97,
110,116,101,101,100,32,116,111,32,98,101,32,100,101,102,105,
110,101,100,32,111,114,32,99,111,117,108,100,32,98,101,32,
115,101,116,32,116,111,32,78,111,110,101,10,32,32,32,32,
116,111,32,114,101,112,114,101,115,101,110,116,32,116,104,97,
116,32,105,116,115,32,112,114,111,112,101,114,32,118,97,108,
117,101,32,105,115,32,117,110,107,110,111,119,110,46,10,10,
32,32,32,32,114,146,0,0,0,114,106,0,0,0,78,122,
32,95,95,112,97,99,107,97,103,101,95,95,32,33,61,32,
95,95,115,112,101,99,95,95,46,112,97,114,101,110,116,32,
40,122,4,32,33,61,32,250,1,41,233,3,0,0,0,41,
1,90,10,115,116,97,99,107,108,101,118,101,108,122,89,99,
97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99,
107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99,
95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95,
95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32,
111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32,
95,95,112,97,116,104,95,95,114,1,0,0,0,114,142,0,
0,0,114,129,0,0,0,114,22,0,0,0,41,6,114,35,
0,0,0,114,131,0,0,0,114,193,0,0,0,114,194,0,
0,0,114,195,0,0,0,114,130,0,0,0,41,3,218,7,
103,108,111,98,97,108,115,114,187,0,0,0,114,96,0,0,
0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
218,17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,
101,95,95,46,4,0,0,115,34,0,0,0,0,7,10,1,
10,1,8,1,18,1,22,2,4,254,6,3,4,1,8,1,
4,0,0,115,52,0,0,0,0,10,8,1,10,1,4,1,
12,2,4,1,4,1,2,255,4,1,8,255,10,2,8,1,
14,1,10,1,2,255,8,2,10,1,14,1,2,1,14,1,
14,4,10,1,16,255,2,2,12,1,26,1,114,215,0,0,
0,99,1,0,0,0,0,0,0,0,0,0,0,0,3,0,
0,0,6,0,0,0,67,0,0,0,115,146,0,0,0,124,
0,160,0,100,1,161,1,125,1,124,0,160,0,100,2,161,
1,125,2,124,1,100,3,117,1,114,82,124,2,100,3,117,
1,114,78,124,1,124,2,106,1,107,3,114,78,116,2,106,
3,100,4,124,1,155,2,100,5,124,2,106,1,155,2,100,
6,157,5,116,4,100,7,100,8,141,3,1,0,124,1,83,
0,124,2,100,3,117,1,114,96,124,2,106,1,83,0,116,
2,106,3,100,9,116,4,100,7,100,8,141,3,1,0,124,
0,100,10,25,0,125,1,100,11,124,0,118,1,114,142,124,
1,160,5,100,12,161,1,100,13,25,0,125,1,124,1,83,
0,41,14,122,167,67,97,108,99,117,108,97,116,101,32,119,
104,97,116,32,95,95,112,97,99,107,97,103,101,95,95,32,
115,104,111,117,108,100,32,98,101,46,10,10,32,32,32,32,
95,95,112,97,99,107,97,103,101,95,95,32,105,115,32,110,
111,116,32,103,117,97,114,97,110,116,101,101,100,32,116,111,
32,98,101,32,100,101,102,105,110,101,100,32,111,114,32,99,
111,117,108,100,32,98,101,32,115,101,116,32,116,111,32,78,
111,110,101,10,32,32,32,32,116,111,32,114,101,112,114,101,
115,101,110,116,32,116,104,97,116,32,105,116,115,32,112,114,
111,112,101,114,32,118,97,108,117,101,32,105,115,32,117,110,
107,110,111,119,110,46,10,10,32,32,32,32,114,146,0,0,
0,114,106,0,0,0,78,122,32,95,95,112,97,99,107,97,
103,101,95,95,32,33,61,32,95,95,115,112,101,99,95,95,
46,112,97,114,101,110,116,32,40,122,4,32,33,61,32,250,
1,41,233,3,0,0,0,41,1,90,10,115,116,97,99,107,
108,101,118,101,108,122,89,99,97,110,39,116,32,114,101,115,
111,108,118,101,32,112,97,99,107,97,103,101,32,102,114,111,
109,32,95,95,115,112,101,99,95,95,32,111,114,32,95,95,
112,97,99,107,97,103,101,95,95,44,32,102,97,108,108,105,
110,103,32,98,97,99,107,32,111,110,32,95,95,110,97,109,
101,95,95,32,97,110,100,32,95,95,112,97,116,104,95,95,
114,1,0,0,0,114,142,0,0,0,114,129,0,0,0,114,
22,0,0,0,41,6,114,35,0,0,0,114,131,0,0,0,
114,193,0,0,0,114,194,0,0,0,114,195,0,0,0,114,
130,0,0,0,41,3,218,7,103,108,111,98,97,108,115,114,
187,0,0,0,114,96,0,0,0,114,10,0,0,0,114,10,
0,0,0,114,11,0,0,0,218,17,95,99,97,108,99,95,
95,95,112,97,99,107,97,103,101,95,95,46,4,0,0,115,
42,0,0,0,0,7,10,1,10,1,8,1,18,1,6,1,
2,255,4,1,4,255,6,2,4,254,6,3,4,1,8,1,
6,2,6,2,4,254,6,3,8,1,8,1,14,1,114,221,
0,0,0,114,10,0,0,0,99,5,0,0,0,0,0,0,
0,0,0,0,0,9,0,0,0,5,0,0,0,67,0,0,

View file

@ -481,10 +481,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
108,101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,
110,97,109,101,114,5,0,0,0,114,5,0,0,0,114,8,
0,0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,
95,99,97,99,104,101,116,1,0,0,115,52,0,0,0,0,
95,99,97,99,104,101,116,1,0,0,115,68,0,0,0,0,
9,12,1,8,1,10,1,12,1,4,1,10,1,12,1,14,
1,16,1,4,1,4,1,12,1,8,1,18,2,10,1,8,
1,16,1,10,1,16,1,10,1,14,2,16,1,10,1,16,
1,16,1,4,1,4,1,12,1,8,1,2,1,2,255,4,
1,2,255,8,2,10,1,8,1,16,1,10,1,16,1,10,
1,4,1,2,255,8,2,16,1,10,1,4,1,2,255,10,
2,14,1,114,102,0,0,0,99,1,0,0,0,0,0,0,
0,0,0,0,0,5,0,0,0,9,0,0,0,67,0,0,
0,115,124,0,0,0,116,0,124,0,131,1,100,1,107,2,

View file

@ -68,6 +68,7 @@ static const char usage_3[] = "\
-X opt : set implementation-specific option. The following options are available:\n\
\n\
-X faulthandler: enable faulthandler\n\
-X oldparser: enable the traditional LL(1) parser; also PYTHONOLDPARSER\n\
-X showrefcount: output the total reference count and number of used\n\
memory blocks when the program finishes or after each statement in the\n\
interactive interpreter. This only works on debug builds\n\
@ -634,6 +635,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
#ifdef MS_WINDOWS
config->legacy_windows_stdio = -1;
#endif
config->use_peg = 1;
}
@ -791,6 +793,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
COPY_ATTR(isolated);
COPY_ATTR(use_environment);
COPY_ATTR(dev_mode);
COPY_ATTR(use_peg);
COPY_ATTR(install_signal_handlers);
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
@ -894,6 +897,7 @@ config_as_dict(const PyConfig *config)
SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
SET_ITEM_INT(dev_mode);
SET_ITEM_INT(use_peg);
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
SET_ITEM_UINT(hash_seed);
@ -1428,6 +1432,11 @@ config_read_complex_options(PyConfig *config)
config->import_time = 1;
}
if (config_get_env(config, "PYTHONOLDPARSER")
|| config_get_xoption(config, L"oldparser")) {
config->use_peg = 0;
}
PyStatus status;
if (config->tracemalloc < 0) {
status = config_init_tracemalloc(config);
@ -2507,6 +2516,7 @@ PyConfig_Read(PyConfig *config)
assert(config->isolated >= 0);
assert(config->use_environment >= 0);
assert(config->dev_mode >= 0);
assert(config->use_peg >= 0);
assert(config->install_signal_handlers >= 0);
assert(config->use_hash_seed >= 0);
assert(config->faulthandler >= 0);

View file

@ -29,6 +29,8 @@
#include "ast.h" // PyAST_FromNodeObject()
#include "marshal.h" // PyMarshal_ReadLongFromFile()
#include <pegen_interface.h> // PyPegen_ASTFrom*
#ifdef MS_WINDOWS
# include "malloc.h" // alloca()
#endif
@ -183,6 +185,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
PyArena *arena;
const char *ps1 = "", *ps2 = "", *enc = NULL;
int errcode = 0;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
_Py_IDENTIFIER(encoding);
_Py_IDENTIFIER(__main__);
@ -235,9 +238,17 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
Py_XDECREF(oenc);
return -1;
}
mod = PyParser_ASTFromFileObject(fp, filename, enc,
Py_single_input, ps1, ps2,
flags, &errcode, arena);
if (use_peg) {
mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input,
enc, ps1, ps2, &errcode, arena);
}
else {
mod = PyParser_ASTFromFileObject(fp, filename, enc,
Py_single_input, ps1, ps2,
flags, &errcode, arena);
}
Py_XDECREF(v);
Py_XDECREF(w);
Py_XDECREF(oenc);
@ -1019,6 +1030,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
mod_ty mod;
PyArena *arena;
PyObject *filename;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
filename = _PyUnicode_FromId(&PyId_string); /* borrowed */
if (filename == NULL)
@ -1028,7 +1040,13 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
if (use_peg) {
mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
}
else {
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
}
if (mod != NULL)
ret = run_mod(mod, filename, globals, locals, flags, arena);
PyArena_Free(arena);
@ -1043,6 +1061,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
mod_ty mod;
PyArena *arena = NULL;
PyObject *filename;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
filename = PyUnicode_DecodeFSDefault(filename_str);
if (filename == NULL)
@ -1052,8 +1071,15 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
if (arena == NULL)
goto exit;
mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
flags, NULL, arena);
if (use_peg) {
mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL,
NULL, arena);
}
else {
mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
flags, NULL, arena);
}
if (closeit)
fclose(fp);
if (mod == NULL) {
@ -1196,11 +1222,17 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start,
{
PyCodeObject *co;
mod_ty mod;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
PyArena *arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
if (use_peg) {
mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
}
else {
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
}
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
@ -1297,13 +1329,19 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py
{
struct symtable *st;
mod_ty mod;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
PyArena *arena;
arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
if (use_peg) {
mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
}
else {
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
}
if (mod == NULL) {
PyArena_Free(arena);
return NULL;

View file

@ -2427,6 +2427,7 @@ static PyStructSequence_Field flags_fields[] = {
{"inspect", "-i"},
{"interactive", "-i"},
{"optimize", "-O or -OO"},
{"use_peg", "-p old or -p new"},
{"dont_write_bytecode", "-B"},
{"no_user_site", "-s"},
{"no_site", "-S"},
@ -2447,7 +2448,7 @@ static PyStructSequence_Desc flags_desc = {
"sys.flags", /* name */
flags__doc__, /* doc */
flags_fields, /* fields */
15
16
};
static PyObject*
@ -2470,6 +2471,7 @@ make_flags(PyThreadState *tstate)
SetFlag(config->inspect);
SetFlag(config->interactive);
SetFlag(config->optimization_level);
SetFlag(config->use_peg);
SetFlag(!config->write_bytecode);
SetFlag(!config->user_site_directory);
SetFlag(!config->site_import);