bpo-40334: Add support for feature_version in new PEG parser (GH-19827)

`ast.parse` and `compile` support a `feature_version` parameter that
tells the parser to parse the input string, as if it were written in
an older Python version.
The `feature_version` is propagated to the tokenizer, which uses it
to handle the three different stages of support for `async` and
`await`. Additionally, it disallows the following at parser level:
- The '@' operator in < 3.5
- Async functions in < 3.5
- Async comprehensions in < 3.6
- Underscores in numeric literals in < 3.6
- Await expression in < 3.5
- Variable annotations in < 3.6
- Async for-loops in < 3.5
- Async with-statements in < 3.5
- F-strings in < 3.6

Closes we-like-parsers/cpython#124.

Parser/pegen/parse_string.c

@@ -179,6 +179,13 @@ _PyPegen_parsestr(Parser *p, const char *s, int *bytesmode, int *rawmode, PyObje
         }
     }
 
+    /* fstrings are only allowed in Python 3.6 and greater */
+    if (fmode && p->feature_version < 6) {
+        p->error_indicator = 1;
+        RAISE_SYNTAX_ERROR("Format strings are only supported in Python 3.6 and greater");
+        return -1;
+    }
+
     if (fmode && *bytesmode) {
         PyErr_BadInternalCall();
         return -1;
@@ -595,7 +602,8 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
         return NULL;
     }
 
-    Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, p->flags, NULL, p->arena);
+    Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, p->flags, p->feature_version,
+                                     NULL, p->arena);
     p2->starting_lineno = p->starting_lineno + p->tok->first_lineno - 1;
     p2->starting_col_offset = p->tok->first_lineno == p->tok->lineno
                               ? p->starting_col_offset + t->col_offset : 0;

Parser/pegen/pegen.c

@@ -933,11 +933,16 @@ _PyPegen_number_token(Parser *p)
     }
 
     char *num_raw = PyBytes_AsString(t->bytes);
-
     if (num_raw == NULL) {
         return NULL;
     }
 
+    if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
+        p->error_indicator = 1;
+        return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported"
+                                  "in Python 3.6 and greater");
+    }
+
     PyObject *c = parsenumber(num_raw);
 
     if (c == NULL) {
@@ -1030,12 +1035,15 @@ compute_parser_flags(PyCompilerFlags *flags)
     if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
         parser_flags |= PyPARSE_TYPE_COMMENTS;
     }
+    if (flags->cf_feature_version < 7) {
+        parser_flags |= PyPARSE_ASYNC_HACKS;
+    }
     return parser_flags;
 }
 
 Parser *
 _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
-                    int *errcode, PyArena *arena)
+                    int feature_version, int *errcode, PyArena *arena)
 {
     Parser *p = PyMem_Malloc(sizeof(Parser));
     if (p == NULL) {
@@ -1077,6 +1085,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
     p->starting_lineno = 0;
     p->starting_col_offset = 0;
     p->flags = flags;
+    p->feature_version = feature_version;
 
     return p;
 }
@@ -1138,7 +1147,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
     mod_ty result = NULL;
 
     int parser_flags = compute_parser_flags(flags);
-    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, errcode, arena);
+    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
+                                    errcode, arena);
     if (p == NULL) {
         goto error;
     }
@@ -1194,9 +1204,12 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
     mod_ty result = NULL;
 
     int parser_flags = compute_parser_flags(flags);
+    int feature_version = flags ? flags->cf_feature_version : PY_MINOR_VERSION;
     tok->type_comments = (parser_flags & PyPARSE_TYPE_COMMENTS) > 0;
+    tok->async_hacks = (parser_flags & PyPARSE_ASYNC_HACKS) > 0;
 
-    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, NULL, arena);
+    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
+                                    NULL, arena);
     if (p == NULL) {
         goto error;
     }

Parser/pegen/pegen.h

@@ -69,6 +69,7 @@ typedef struct {
     int starting_col_offset;
     int error_indicator;
     int flags;
+    int feature_version;
     growable_comment_array type_ignore_comments;
 } Parser;
 
@@ -180,9 +181,26 @@ NEW_TYPE_COMMENT(Parser *p, Token *tc)
     return NULL;
 }
 
+Py_LOCAL_INLINE(void *)
+INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
+{
+    if (node == NULL) {
+        p->error_indicator = 1;  // Inline CHECK_CALL
+        return NULL;
+    }
+    if (p->feature_version < version) {
+        p->error_indicator = 1;
+        return _PyPegen_raise_error(p, PyExc_SyntaxError, "%s only supported in Python 3.%i and greater",
+                                    msg, version);
+    }
+    return node;
+}
+
+#define CHECK_VERSION(version, msg, node) INVALID_VERSION_CHECK(p, version, msg, node)
+
 arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
 PyObject *_PyPegen_new_identifier(Parser *, char *);
-Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int *, PyArena *);
+Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
 void _PyPegen_Parser_Free(Parser *);
 mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
                                     const char *, const char *, PyCompilerFlags *, int *, PyArena *);