mirror of
https://github.com/python/cpython.git
synced 2025-09-26 10:19:53 +00:00
bpo-40334: Add support for feature_version in new PEG parser (GH-19827)
`ast.parse` and `compile` support a `feature_version` parameter that tells the parser to parse the input string, as if it were written in an older Python version. The `feature_version` is propagated to the tokenizer, which uses it to handle the three different stages of support for `async` and `await`. Additionally, it disallows the following at parser level: - The '@' operator in < 3.5 - Async functions in < 3.5 - Async comprehensions in < 3.6 - Underscores in numeric literals in < 3.6 - Await expression in < 3.5 - Variable annotations in < 3.6 - Async for-loops in < 3.5 - Async with-statements in < 3.5 - F-strings in < 3.6 Closes we-like-parsers/cpython#124.
This commit is contained in:
Lysandros Nikolaou
GitHub
parent
eb0d359b4b
commit
3e0a6f37df
6 changed files with 1970 additions and 1425 deletions
|
|
@ -80,10 +80,14 @@ compound_stmt[stmt_ty]:
|
|||
# NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield'
|
||||
assignment:
|
||||
| a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] {
|
||||
_Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) }
|
||||
CHECK_VERSION(
|
||||
6,
|
||||
"Variable annotation syntax is",
|
||||
_Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA)
|
||||
) }
|
||||
| a=('(' b=inside_paren_ann_assign_target ')' { b }
|
||||
| ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] {
|
||||
_Py_AnnAssign(a, b, c, 0, EXTRA)}
|
||||
CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) }
|
||||
| a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] {
|
||||
_Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| a=target b=augassign c=(yield_expr | star_expressions) {
|
||||
|
|
@ -91,19 +95,19 @@ assignment:
|
|||
| invalid_assignment
|
||||
|
||||
augassign[AugOperator*]:
|
||||
| '+=' {_PyPegen_augoperator(p, Add)}
|
||||
| '-=' {_PyPegen_augoperator(p, Sub)}
|
||||
| '*=' {_PyPegen_augoperator(p, Mult)}
|
||||
| '@=' {_PyPegen_augoperator(p, MatMult)}
|
||||
| '/=' {_PyPegen_augoperator(p, Div)}
|
||||
| '%=' {_PyPegen_augoperator(p, Mod)}
|
||||
| '&=' {_PyPegen_augoperator(p, BitAnd)}
|
||||
| '|=' {_PyPegen_augoperator(p, BitOr)}
|
||||
| '^=' {_PyPegen_augoperator(p, BitXor)}
|
||||
| '<<=' {_PyPegen_augoperator(p, LShift)}
|
||||
| '>>=' {_PyPegen_augoperator(p, RShift)}
|
||||
| '**=' {_PyPegen_augoperator(p, Pow)}
|
||||
| '//=' {_PyPegen_augoperator(p, FloorDiv)}
|
||||
| '+=' { _PyPegen_augoperator(p, Add) }
|
||||
| '-=' { _PyPegen_augoperator(p, Sub) }
|
||||
| '*=' { _PyPegen_augoperator(p, Mult) }
|
||||
| '@=' { CHECK_VERSION(5, "The '@' operator is", _PyPegen_augoperator(p, MatMult)) }
|
||||
| '/=' { _PyPegen_augoperator(p, Div) }
|
||||
| '%=' { _PyPegen_augoperator(p, Mod) }
|
||||
| '&=' { _PyPegen_augoperator(p, BitAnd) }
|
||||
| '|=' { _PyPegen_augoperator(p, BitOr) }
|
||||
| '^=' { _PyPegen_augoperator(p, BitXor) }
|
||||
| '<<=' { _PyPegen_augoperator(p, LShift) }
|
||||
| '>>=' { _PyPegen_augoperator(p, RShift) }
|
||||
| '**=' { _PyPegen_augoperator(p, Pow) }
|
||||
| '//=' { _PyPegen_augoperator(p, FloorDiv) }
|
||||
|
||||
global_stmt[stmt_ty]: 'global' a=','.NAME+ {
|
||||
_Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) }
|
||||
|
|
@ -156,14 +160,20 @@ while_stmt[stmt_ty]:
|
|||
| 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) }
|
||||
|
||||
for_stmt[stmt_ty]:
|
||||
| is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
|
||||
(is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
|
||||
_Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| ASYNC 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
|
||||
CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
|
||||
|
||||
with_stmt[stmt_ty]:
|
||||
| is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block {
|
||||
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) }
|
||||
| is_async=[ASYNC] 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
|
||||
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| 'with' '(' a=','.with_item+ ')' ':' b=block {
|
||||
_Py_With(a, b, NULL, EXTRA) }
|
||||
| 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
|
||||
_Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| ASYNC 'with' '(' a=','.with_item+ ')' ':' b=block {
|
||||
CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) }
|
||||
| ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
|
||||
CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
|
||||
with_item[withitem_ty]:
|
||||
| e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) }
|
||||
|
||||
|
|
@ -188,10 +198,18 @@ function_def[stmt_ty]:
|
|||
| function_def_raw
|
||||
|
||||
function_def_raw[stmt_ty]:
|
||||
| is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
|
||||
(is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id,
|
||||
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
|
||||
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
|
||||
_Py_FunctionDef(n->v.Name.id,
|
||||
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
|
||||
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
|
||||
| ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
|
||||
CHECK_VERSION(
|
||||
5,
|
||||
"Async functions are",
|
||||
_Py_AsyncFunctionDef(n->v.Name.id,
|
||||
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
|
||||
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA)
|
||||
) }
|
||||
func_type_comment[PyObject*]:
|
||||
| NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t } # Must be followed by indented block
|
||||
| invalid_double_type_comments
|
||||
|
|
@ -399,7 +417,7 @@ term[expr_ty]:
|
|||
| a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) }
|
||||
| a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) }
|
||||
| a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) }
|
||||
| a=term '@' b=factor { _Py_BinOp(a, MatMult, b, EXTRA) }
|
||||
| a=term '@' b=factor { CHECK_VERSION(5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) }
|
||||
| factor
|
||||
factor[expr_ty] (memo):
|
||||
| '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) }
|
||||
|
|
@ -410,7 +428,7 @@ power[expr_ty]:
|
|||
| a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) }
|
||||
| await_primary
|
||||
await_primary[expr_ty] (memo):
|
||||
| AWAIT a=primary { _Py_Await(a, EXTRA) }
|
||||
| AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) }
|
||||
| primary
|
||||
primary[expr_ty]:
|
||||
| a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) }
|
||||
|
|
@ -469,8 +487,12 @@ kvpair[KeyValuePair*]:
|
|||
| '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) }
|
||||
| a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) }
|
||||
for_if_clauses[asdl_seq*]:
|
||||
| a=(y=[ASYNC] 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })*
|
||||
{ _Py_comprehension(a, b, c, y != NULL, p->arena) })+ { a }
|
||||
| for_if_clause+
|
||||
for_if_clause[comprehension_ty]:
|
||||
| ASYNC 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
|
||||
CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) }
|
||||
| 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
|
||||
_Py_comprehension(a, b, c, 0, p->arena) }
|
||||
|
||||
yield_expr[expr_ty]:
|
||||
| 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue