mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-21 14:12:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-35975: Support parsing earlier minor versions of Python 3 (GH-12086)

Browse source 
This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.)



https://bugs.python.org/issue35975
This commit is contained in:
Guido van Rossum 2019-03-07 12:38:08 -08:00 • committed by Miss Islington (bot)
parent bf94cc7b49
commit 495da29225
29 changed files with 476 additions and 201 deletions
Download patch file Download diff file Expand all files Collapse all files

100
Python/ast.c
View file

@ -564,6 +564,7 @@ struct compiling {
PyArena *c_arena; /* Arena for allocating memory. */
PyObject *c_filename; /* filename */
PyObject *c_normalize; /* Normalization function from unicodedata. */
int c_feature_version; /* Latest minor version of Python for allowed features */
};
static asdl_seq *seq_for_testlist(struct compiling *, const node *);
@ -783,6 +784,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
/* borrowed reference */
c.c_filename = filename;
c.c_normalize = NULL;
c.c_feature_version = flags->cf_feature_version;
if (TYPE(n) == encoding_decl)
n = CHILD(n, 0);
@ -955,7 +957,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str,
*/
static operator_ty
get_operator(const node *n)
get_operator(struct compiling *c, const node *n)
{
switch (TYPE(n)) {
case VBAR:
@ -975,6 +977,11 @@ get_operator(const node *n)
case STAR:
return Mult;
case AT:
if (c->c_feature_version < 5) {
ast_error(c, n,
"The '@' operator is only supported in Python 3.5 and greater");
return (operator_ty)0;
}
return MatMult;
case SLASH:
return Div;
@ -1209,6 +1216,11 @@ ast_for_augassign(struct compiling *c, const node *n)
else
return Mult;
case '@':
if (c->c_feature_version < 5) {
ast_error(c, n,
"The '@' operator is only supported in Python 3.5 and greater");
return (operator_ty)0;
}
return MatMult;
default:
PyErr_Format(PyExc_SystemError, "invalid augassign: %s", STR(n));
@ -1518,7 +1530,7 @@ ast_for_arguments(struct compiling *c, const node *n)
}
else if (found_default) {
ast_error(c, n,
"non-default argument follows default argument");
"non-default argument follows default argument");
return NULL;
}
arg = ast_for_arg(c, ch);
@ -1719,6 +1731,12 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
node *tc;
string type_comment = NULL;
if (is_async && c->c_feature_version < 5) {
ast_error(c, n,
"Async functions are only supported in Python 3.5 and greater");
return NULL;
}
REQ(n, funcdef);
name = NEW_IDENTIFIER(CHILD(n, name_i));
@ -1772,10 +1790,9 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
static stmt_ty
ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
/* async_funcdef: 'async' funcdef */
/* async_funcdef: ASYNC funcdef */
REQ(n, async_funcdef);
REQ(CHILD(n, 0), NAME);
assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
REQ(CHILD(n, 0), ASYNC);
REQ(CHILD(n, 1), funcdef);
return ast_for_funcdef_impl(c, n, decorator_seq,
@ -1794,10 +1811,9 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
static stmt_ty
ast_for_async_stmt(struct compiling *c, const node *n)
{
/* async_stmt: 'async' (funcdef | with_stmt | for_stmt) */
/* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
REQ(n, async_stmt);
REQ(CHILD(n, 0), NAME);
assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
REQ(CHILD(n, 0), ASYNC);
switch (TYPE(CHILD(n, 1))) {
case funcdef:
@ -1948,8 +1964,7 @@ count_comp_fors(struct compiling *c, const node *n)
n_fors++;
REQ(n, comp_for);
if (NCH(n) == 2) {
REQ(CHILD(n, 0), NAME);
assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
REQ(CHILD(n, 0), ASYNC);
n = CHILD(n, 1);
}
else if (NCH(n) == 1) {
@ -2034,8 +2049,7 @@ ast_for_comprehension(struct compiling *c, const node *n)
if (NCH(n) == 2) {
is_async = 1;
REQ(CHILD(n, 0), NAME);
assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
REQ(CHILD(n, 0), ASYNC);
sync_n = CHILD(n, 1);
}
else {
@ -2043,6 +2057,13 @@ ast_for_comprehension(struct compiling *c, const node *n)
}
REQ(sync_n, sync_comp_for);
/* Async comprehensions only allowed in Python 3.6 and greater */
if (is_async && c->c_feature_version < 6) {
ast_error(c, n,
"Async comprehensions are only supported in Python 3.6 and greater");
return NULL;
}
for_ch = CHILD(sync_n, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
@ -2337,7 +2358,15 @@ ast_for_atom(struct compiling *c, const node *n)
return str;
}
case NUMBER: {
PyObject *pynum = parsenumber(c, STR(ch));
PyObject *pynum;
/* Underscores in numeric literals are only allowed in Python 3.6 or greater */
/* Check for underscores here rather than in parse_number so we can report a line number on error */
if (c->c_feature_version < 6 && strchr(STR(ch), '_') != NULL) {
ast_error(c, ch,
"Underscores in numeric literals are only supported in Python 3.6 and greater");
return NULL;
}
pynum = parsenumber(c, STR(ch));
if (!pynum)
return NULL;
@ -2420,8 +2449,8 @@ ast_for_atom(struct compiling *c, const node *n)
TYPE(CHILD(ch, 3 - is_dict)) == comp_for) {
/* It's a dictionary comprehension. */
if (is_dict) {
ast_error(c, n, "dict unpacking cannot be used in "
"dict comprehension");
ast_error(c, n,
"dict unpacking cannot be used in dict comprehension");
return NULL;
}
res = ast_for_dictcomp(c, ch);
@ -2524,7 +2553,7 @@ ast_for_binop(struct compiling *c, const node *n)
if (!expr2)
return NULL;
newoperator = get_operator(CHILD(n, 1));
newoperator = get_operator(c, CHILD(n, 1));
if (!newoperator)
return NULL;
@ -2539,7 +2568,7 @@ ast_for_binop(struct compiling *c, const node *n)
expr_ty tmp_result, tmp;
const node* next_oper = CHILD(n, i * 2 + 1);
newoperator = get_operator(next_oper);
newoperator = get_operator(c, next_oper);
if (!newoperator)
return NULL;
@ -2678,7 +2707,12 @@ ast_for_atom_expr(struct compiling *c, const node *n)
REQ(n, atom_expr);
nch = NCH(n);
if (TYPE(CHILD(n, 0)) == NAME && strcmp(STR(CHILD(n, 0)), "await") == 0) {
if (TYPE(CHILD(n, 0)) == AWAIT) {
if (c->c_feature_version < 5) {
ast_error(c, n,
"Await expressions are only supported in Python 3.5 and greater");
return NULL;
}
start = 1;
assert(nch > 1);
}
@ -2775,7 +2809,7 @@ ast_for_expr(struct compiling *c, const node *n)
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom_expr ['**' factor]
atom_expr: ['await'] atom trailer*
atom_expr: [AWAIT] atom trailer*
yield_expr: 'yield' [yield_arg]
*/
@ -3233,6 +3267,13 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
node *deep, *ann = CHILD(n, 1);
int simple = 1;
/* AnnAssigns are only allowed in Python 3.6 or greater */
if (c->c_feature_version < 6) {
ast_error(c, ch,
"Variable annotation syntax is only supported in Python 3.6 and greater");
return NULL;
}
/* we keep track of parens to qualify (x) as expression not name */
deep = ch;
while (NCH(deep) == 1) {
@ -4050,6 +4091,13 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
int end_lineno, end_col_offset;
int has_type_comment;
string type_comment;
if (is_async && c->c_feature_version < 5) {
ast_error(c, n,
"Async for loops are only supported in Python 3.5 and greater");
return NULL;
}
/* for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] */
REQ(n, for_stmt);
@ -4278,6 +4326,12 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
asdl_seq *items, *body;
string type_comment;
if (is_async && c->c_feature_version < 5) {
ast_error(c, n,
"Async with statements are only supported in Python 3.5 and greater");
return NULL;
}
REQ(n, with_stmt);
has_type_comment = TYPE(CHILD(n, NCH(n) - 2)) == TYPE_COMMENT;
@ -4768,6 +4822,7 @@ fstring_compile_expr(const char *expr_start, const char *expr_end,
str[len+2] = 0;
cf.cf_flags = PyCF_ONLY_AST;
cf.cf_feature_version = PY_MINOR_VERSION;
mod_n = PyParser_SimpleParseStringFlagsFilename(str, "<fstring>",
Py_eval_input, 0);
if (!mod_n) {
@ -5568,6 +5623,13 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *rawmode,
}
}
}
/* fstrings are only allowed in Python 3.6 and greater */
if (fmode && c->c_feature_version < 6) {
ast_error(c, n, "Format strings are only supported in Python 3.6 and greater");
return -1;
}
if (fmode && *bytesmode) {
PyErr_BadInternalCall();
return -1;