mirror of
https://github.com/python/cpython.git
synced 2025-08-04 17:08:35 +00:00
Refactor parser compilation units into specific components (GH-29676)
This commit is contained in:
Pablo Galindo Salgado
GitHub
parent
f7638dd0f9
commit
c9c4444d9f
10 changed files with 1910 additions and 1870 deletions
112
Parser/pegen.h
112
Parser/pegen.h
|
|
@ -23,6 +23,8 @@
|
|||
#define PyPARSE_TYPE_COMMENTS 0x0040
|
||||
#define PyPARSE_ASYNC_HACKS 0x0080
|
||||
|
||||
#define CURRENT_POS (-5)
|
||||
|
||||
typedef struct _memo {
|
||||
int type;
|
||||
void *node;
|
||||
|
|
@ -114,6 +116,7 @@ typedef struct {
|
|||
int is_keyword;
|
||||
} KeywordOrStarred;
|
||||
|
||||
// Internal parser functions
|
||||
#if defined(Py_DEBUG)
|
||||
void _PyPegen_clear_memo_statistics(void);
|
||||
PyObject *_PyPegen_get_memo_statistics(void);
|
||||
|
|
@ -123,7 +126,6 @@ int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node);
|
|||
int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
|
||||
int _PyPegen_is_memoized(Parser *p, int type, void *pres);
|
||||
|
||||
|
||||
int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
|
||||
int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
|
||||
int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*);
|
||||
|
|
@ -139,23 +141,24 @@ int _PyPegen_fill_token(Parser *p);
|
|||
expr_ty _PyPegen_name_token(Parser *p);
|
||||
expr_ty _PyPegen_number_token(Parser *p);
|
||||
void *_PyPegen_string_token(Parser *p);
|
||||
const char *_PyPegen_get_expr_name(expr_ty);
|
||||
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
|
||||
|
||||
// Error handling functions and APIs
|
||||
typedef enum {
|
||||
STAR_TARGETS,
|
||||
DEL_TARGETS,
|
||||
FOR_TARGETS
|
||||
} TARGETS_TYPE;
|
||||
|
||||
int _Pypegen_raise_decode_error(Parser *p);
|
||||
void _PyPegen_raise_tokenizer_init_error(PyObject *filename);
|
||||
int _Pypegen_tokenizer_error(Parser *p);
|
||||
void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
|
||||
void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
|
||||
Py_ssize_t lineno, Py_ssize_t col_offset,
|
||||
Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
|
||||
const char *errmsg, va_list va);
|
||||
void *_PyPegen_dummy_name(Parser *p, ...);
|
||||
|
||||
void * _PyPegen_seq_last_item(asdl_seq *seq);
|
||||
#define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
|
||||
|
||||
void * _PyPegen_seq_first_item(asdl_seq *seq);
|
||||
#define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
|
||||
|
||||
#define CURRENT_POS (-5)
|
||||
|
||||
void _Pypegen_set_syntax_error(Parser* p, Token* last_token);
|
||||
Py_LOCAL_INLINE(void *)
|
||||
RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
|
||||
Py_ssize_t lineno, Py_ssize_t col_offset,
|
||||
|
|
@ -170,10 +173,6 @@ RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
|
|||
va_end(va);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#define UNUSED(expr) do { (void)(expr); } while (0)
|
||||
#define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
|
||||
#define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
|
||||
#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__)
|
||||
#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__)
|
||||
#define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \
|
||||
|
|
@ -182,6 +181,7 @@ RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
|
|||
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__)
|
||||
#define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \
|
||||
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__)
|
||||
#define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
|
||||
|
||||
Py_LOCAL_INLINE(void *)
|
||||
CHECK_CALL(Parser *p, void *result)
|
||||
|
|
@ -207,6 +207,39 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
|
|||
#define CHECK(type, result) ((type) CHECK_CALL(p, result))
|
||||
#define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result))
|
||||
|
||||
expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
|
||||
const char *_PyPegen_get_expr_name(expr_ty);
|
||||
Py_LOCAL_INLINE(void *)
|
||||
_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
|
||||
{
|
||||
expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
|
||||
if (invalid_target != NULL) {
|
||||
const char *msg;
|
||||
if (type == STAR_TARGETS || type == FOR_TARGETS) {
|
||||
msg = "cannot assign to %s";
|
||||
}
|
||||
else {
|
||||
msg = "cannot delete %s";
|
||||
}
|
||||
return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
invalid_target,
|
||||
msg,
|
||||
_PyPegen_get_expr_name(invalid_target)
|
||||
);
|
||||
}
|
||||
return RAISE_SYNTAX_ERROR("invalid syntax");
|
||||
}
|
||||
|
||||
// Action utility functions
|
||||
|
||||
void *_PyPegen_dummy_name(Parser *p, ...);
|
||||
void * _PyPegen_seq_last_item(asdl_seq *seq);
|
||||
#define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
|
||||
void * _PyPegen_seq_first_item(asdl_seq *seq);
|
||||
#define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
|
||||
#define UNUSED(expr) do { (void)(expr); } while (0)
|
||||
#define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
|
||||
#define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
|
||||
PyObject *_PyPegen_new_type_comment(Parser *, const char *);
|
||||
|
||||
Py_LOCAL_INLINE(PyObject *)
|
||||
|
|
@ -248,13 +281,6 @@ INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
|
|||
|
||||
arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
|
||||
PyObject *_PyPegen_new_identifier(Parser *, const char *);
|
||||
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
|
||||
void _PyPegen_Parser_Free(Parser *);
|
||||
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
|
||||
const char *, const char *, PyCompilerFlags *, int *, PyArena *);
|
||||
void *_PyPegen_run_parser(Parser *);
|
||||
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
|
||||
asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
|
||||
asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
|
||||
asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
|
||||
asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
|
||||
|
|
@ -295,40 +321,18 @@ asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
|
|||
int _PyPegen_check_barry_as_flufl(Parser *, Token *);
|
||||
int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t);
|
||||
mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *);
|
||||
|
||||
// Error reporting helpers
|
||||
typedef enum {
|
||||
STAR_TARGETS,
|
||||
DEL_TARGETS,
|
||||
FOR_TARGETS
|
||||
} TARGETS_TYPE;
|
||||
expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
|
||||
#define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
|
||||
|
||||
Py_LOCAL_INLINE(void *)
|
||||
_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
|
||||
{
|
||||
expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
|
||||
if (invalid_target != NULL) {
|
||||
const char *msg;
|
||||
if (type == STAR_TARGETS || type == FOR_TARGETS) {
|
||||
msg = "cannot assign to %s";
|
||||
}
|
||||
else {
|
||||
msg = "cannot delete %s";
|
||||
}
|
||||
return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
|
||||
invalid_target,
|
||||
msg,
|
||||
_PyPegen_get_expr_name(invalid_target)
|
||||
);
|
||||
}
|
||||
return RAISE_SYNTAX_ERROR("invalid syntax");
|
||||
}
|
||||
|
||||
void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
|
||||
void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
|
||||
|
||||
// Parser API
|
||||
|
||||
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
|
||||
void _PyPegen_Parser_Free(Parser *);
|
||||
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
|
||||
const char *, const char *, PyCompilerFlags *, int *, PyArena *);
|
||||
void *_PyPegen_run_parser(Parser *);
|
||||
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
|
||||
asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
|
||||
|
||||
// Generated function in parse.c - function definition in python.gram
|
||||
void *_PyPegen_parse(Parser *);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue