mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-19 09:15:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

gh-119180: PEP 649 compiler changes (#119361)

Browse source
This commit is contained in:
Jelle Zijlstra 2024-06-11 07:06:49 -06:00 • committed by GitHub
parent 02c1dfff07
commit 9b8611eeea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 610 additions and 329 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Python/bytecodes.c
View file

@ -3975,6 +3975,11 @@ dummy_func(
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
case MAKE_FUNCTION_ANNOTATE:
assert(PyCallable_Check(attr));
assert(func_obj->func_annotate == NULL);
func_obj->func_annotate = attr;
break;
default:
Py_UNREACHABLE();
}

359
Python/compile.c
View file

@ -132,7 +132,7 @@ enum {
COMPILER_SCOPE_ASYNC_FUNCTION,
COMPILER_SCOPE_LAMBDA,
COMPILER_SCOPE_COMPREHENSION,
COMPILER_SCOPE_TYPEPARAMS,
COMPILER_SCOPE_ANNOTATIONS,
};
@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence;
#define INITIAL_INSTR_SEQUENCE_SIZE 100
#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10
static const int compare_masks[] = {
[Py_LT] = COMPARISON_LESS_THAN,
[Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
[Py_EQ] = COMPARISON_EQUALS,
[Py_NE] = COMPARISON_NOT_EQUALS,
[Py_GT] = COMPARISON_GREATER_THAN,
[Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
};
/*
* Resize the array if index is out of range.
*
@ -208,6 +217,7 @@ struct compiler_unit {
PyObject *u_private; /* for private name mangling */
PyObject *u_static_attributes; /* for class: attributes accessed via self.X */
PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */
instr_sequence *u_instr_sequence; /* codegen output */
@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *);
static int compiler_match(struct compiler *, stmt_ty);
static int compiler_pattern_subpattern(struct compiler *,
pattern_ty, pattern_context *);
static int compiler_make_closure(struct compiler *c, location loc,
PyCodeObject *co, Py_ssize_t flags);
static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone);
@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u)
Py_CLEAR(u->u_metadata.u_fasthidden);
Py_CLEAR(u->u_private);
Py_CLEAR(u->u_static_attributes);
Py_CLEAR(u->u_deferred_annotations);
PyMem_Free(u);
}
@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c)
capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
assert(parent);
if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) {
/* The parent is a type parameter scope, so we need to
if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) {
/* The parent is an annotation scope, so we need to
look at the grandparent. */
if (stack_size == 2) {
// If we're immediately within the module, we can skip
@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name,
}
u->u_private = NULL;
u->u_deferred_annotations = NULL;
if (scope_type == COMPILER_SCOPE_CLASS) {
u->u_static_attributes = PySet_New(0);
if (!u->u_static_attributes) {
@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c)
PyErr_SetRaisedException(exc);
}
/* Search if variable annotations are present statically in a block. */
static bool
find_ann(asdl_stmt_seq *stmts)
{
int i, j, res = 0;
stmt_ty st;
for (i = 0; i < asdl_seq_LEN(stmts); i++) {
st = (stmt_ty)asdl_seq_GET(stmts, i);
switch (st->kind) {
case AnnAssign_kind:
return true;
case For_kind:
res = find_ann(st->v.For.body) ||
find_ann(st->v.For.orelse);
break;
case AsyncFor_kind:
res = find_ann(st->v.AsyncFor.body) ||
find_ann(st->v.AsyncFor.orelse);
break;
case While_kind:
res = find_ann(st->v.While.body) ||
find_ann(st->v.While.orelse);
break;
case If_kind:
res = find_ann(st->v.If.body) ||
find_ann(st->v.If.orelse);
break;
case With_kind:
res = find_ann(st->v.With.body);
break;
case AsyncWith_kind:
res = find_ann(st->v.AsyncWith.body);
break;
case Try_kind:
for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) {
excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
st->v.Try.handlers, j);
if (find_ann(handler->v.ExceptHandler.body)) {
return true;
}
}
res = find_ann(st->v.Try.body) ||
find_ann(st->v.Try.finalbody) ||
find_ann(st->v.Try.orelse);
break;
case TryStar_kind:
for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) {
excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
st->v.TryStar.handlers, j);
if (find_ann(handler->v.ExceptHandler.body)) {
return true;
}
}
res = find_ann(st->v.TryStar.body) ||
find_ann(st->v.TryStar.finalbody) ||
find_ann(st->v.TryStar.orelse);
break;
case Match_kind:
for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) {
match_case_ty match_case = (match_case_ty)asdl_seq_GET(
st->v.Match.cases, j);
if (find_ann(match_case->body)) {
return true;
}
}
break;
default:
res = false;
break;
}
if (res) {
break;
}
}
return res;
}
/*
* Frame block handling functions
*/
@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc,
return SUCCESS;
}
static int
compiler_setup_annotations_scope(struct compiler *c, location loc,
void *key, PyObject *name)
{
if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
key, loc.lineno) == -1) {
return ERROR;
}
c->u->u_metadata.u_posonlyargcount = 1;
// if .format != 1: raise NotImplementedError
_Py_DECLARE_STR(format, ".format");
ADDOP_I(c, loc, LOAD_FAST, 0);
ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne());
ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]);
NEW_JUMP_TARGET_LABEL(c, body);
ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);
ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR);
ADDOP_I(c, loc, RAISE_VARARGS, 1);
USE_LABEL(c, body);
return 0;
}
static int
compiler_leave_annotations_scope(struct compiler *c, location loc,
Py_ssize_t annotations_len)
{
ADDOP_I(c, loc, BUILD_MAP, annotations_len);
ADDOP_IN_SCOPE(c, loc, RETURN_VALUE);
PyCodeObject *co = optimize_and_assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL) {
return ERROR;
}
if (compiler_make_closure(c, loc, co, 0) < 0) {
Py_DECREF(co);
return ERROR;
}
Py_DECREF(co);
return 0;
}
/* Compile a sequence of statements, checking for a docstring
and for annotations. */
@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts)
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
loc = LOC(st);
}
/* Every annotated class and module should have __annotations__. */
if (find_ann(stmts)) {
/* If from __future__ import annotations is active,
* every annotated class and module should have __annotations__.
* Else __annotate__ is created when necessary. */
if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) {
ADDOP(c, loc, SETUP_ANNOTATIONS);
}
if (!asdl_seq_LEN(stmts)) {
return SUCCESS;
}
Py_ssize_t first_instr = 0;
PyObject *docstring = _PyAST_GetDocString(stmts);
if (docstring) {
first_instr = 1;
/* if not -OO mode, set docstring */
if (c->c_optimize < 2) {
PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
if (cleandoc == NULL) {
return ERROR;
if (!c->c_interactive) {
PyObject *docstring = _PyAST_GetDocString(stmts);
if (docstring) {
first_instr = 1;
/* if not -OO mode, set docstring */
if (c->c_optimize < 2) {
PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
if (cleandoc == NULL) {
return ERROR;
}
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
assert(st->kind == Expr_kind);
location loc = LOC(st->v.Expr.value);
ADDOP_LOAD_CONST(c, loc, cleandoc);
Py_DECREF(cleandoc);
RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
}
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
assert(st->kind == Expr_kind);
location loc = LOC(st->v.Expr.value);
ADDOP_LOAD_CONST(c, loc, cleandoc);
Py_DECREF(cleandoc);
RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
}
}
for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) {
VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
}
// If there are annotations and the future import is not on, we
// collect the annotations in a separate pass and generate an
// __annotate__ function. See PEP 649.
if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) &&
c->u->u_deferred_annotations != NULL) {
// It's possible that ste_annotations_block is set but
// u_deferred_annotations is not, because the former is still
// set if there are only non-simple annotations (i.e., annotations
// for attributes, subscripts, or parenthesized names). However, the
// reverse should not be possible.
assert(c->u->u_ste->ste_annotation_block != NULL);
PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations);
void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1);
if (compiler_setup_annotations_scope(c, loc, key,
c->u->u_ste->ste_annotation_block->ste_name) == -1) {
Py_DECREF(deferred_anno);
return ERROR;
}
Py_ssize_t annotations_len = PyList_Size(deferred_anno);
for (Py_ssize_t i = 0; i < annotations_len; i++) {
PyObject *ptr = PyList_GET_ITEM(deferred_anno, i);
stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr);
if (st == NULL) {
compiler_exit_scope(c);
Py_DECREF(deferred_anno);
return ERROR;
}
PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id);
ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled);
VISIT(c, expr, st->v.AnnAssign.annotation);
}
Py_DECREF(deferred_anno);
RETURN_IF_ERROR(
compiler_leave_annotations_scope(c, loc, annotations_len)
);
RETURN_IF_ERROR(
compiler_nameop(c, loc, &_Py_ID(__annotate__), Store)
);
}
return SUCCESS;
}
@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod)
}
break;
case Interactive_kind:
if (find_ann(mod->v.Interactive.body)) {
ADDOP(c, loc, SETUP_ANNOTATIONS);
}
c->c_interactive = 1;
VISIT_SEQ(c, stmt, mod->v.Interactive.body);
if (compiler_body(c, loc, mod->v.Interactive.body) < 0) {
return ERROR;
}
break;
case Expression_kind:
VISIT(c, expr, mod->v.Expression.body);
@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc,
if (flags & MAKE_FUNCTION_ANNOTATIONS) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS);
}
if (flags & MAKE_FUNCTION_ANNOTATE) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE);
}
if (flags & MAKE_FUNCTION_KWDEFAULTS) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS);
}
@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
VISIT(c, expr, annotation);
}
}
*annotations_len += 2;
*annotations_len += 1;
return SUCCESS;
}
@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args,
}
static int
compiler_visit_annotations(struct compiler *c, location loc,
arguments_ty args, expr_ty returns)
compiler_visit_annotations_in_scope(struct compiler *c, location loc,
arguments_ty args, expr_ty returns,
Py_ssize_t *annotations_len)
{
/* Push arg annotation names and values.
The expressions are evaluated out-of-order wrt the source code.
Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed.
*/
Py_ssize_t annotations_len = 0;
RETURN_IF_ERROR(
compiler_visit_argannotations(c, args->args, annotations_len, loc));
RETURN_IF_ERROR(
compiler_visit_argannotations(c, args->args, &annotations_len, loc));
RETURN_IF_ERROR(
compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc));
compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc));
if (args->vararg && args->vararg->annotation) {
RETURN_IF_ERROR(
compiler_visit_argannotation(c, args->vararg->arg,
args->vararg->annotation, &annotations_len, loc));
args->vararg->annotation, annotations_len, loc));
}
RETURN_IF_ERROR(
compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc));
compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc));
if (args->kwarg && args->kwarg->annotation) {
RETURN_IF_ERROR(
compiler_visit_argannotation(c, args->kwarg->arg,
args->kwarg->annotation, &annotations_len, loc));
args->kwarg->annotation, annotations_len, loc));
}
RETURN_IF_ERROR(
compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc));
compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc));
if (annotations_len) {
ADDOP_I(c, loc, BUILD_TUPLE, annotations_len);
return 1;
return 0;
}
static int
compiler_visit_annotations(struct compiler *c, location loc,
arguments_ty args, expr_ty returns)
{
/* Push arg annotation names and values.
The expressions are evaluated separately from the rest of the source code.
Return -1 on error, or a combination of flags to add to the function.
*/
Py_ssize_t annotations_len = 0;
PySTEntryObject *ste;
if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) {
return ERROR;
}
assert(ste != NULL);
bool annotations_used = ste->ste_annotations_used;
if (annotations_used) {
if (compiler_setup_annotations_scope(c, loc, (void *)args,
ste->ste_name) < 0) {
Py_DECREF(ste);
return ERROR;
}
}
Py_DECREF(ste);
if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) {
if (annotations_used) {
compiler_exit_scope(c);
}
return ERROR;
}
if (annotations_used) {
RETURN_IF_ERROR(
compiler_leave_annotations_scope(c, loc, annotations_len)
);
return MAKE_FUNCTION_ANNOTATE;
}
return 0;
@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e,
identifier name, void *key,
bool allow_starred)
{
if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS,
if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
key, e->lineno) == -1) {
return ERROR;
}
@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
asdl_expr_seq *decos;
asdl_type_param_seq *type_params;
Py_ssize_t funcflags;
int annotations;
int firstlineno;
if (is_async) {
@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
if (!type_params_name) {
return ERROR;
}
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
(void *)type_params, firstlineno) == -1) {
Py_DECREF(type_params_name);
return ERROR;
@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
}
}
annotations = compiler_visit_annotations(c, loc, args, returns);
if (annotations < 0) {
int annotations_flag = compiler_visit_annotations(c, loc, args, returns);
if (annotations_flag < 0) {
if (is_generic) {
compiler_exit_scope(c);
}
return ERROR;
}
if (annotations > 0) {
funcflags |= MAKE_FUNCTION_ANNOTATIONS;
}
funcflags |= annotations_flag;
if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) {
if (is_generic) {
@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s)
if (!type_params_name) {
return ERROR;
}
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
(void *)type_params, firstlineno) == -1) {
Py_DECREF(type_params_name);
return ERROR;
@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s)
if (!type_params_name) {
return ERROR;
}
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
(void *)type_params, loc.lineno) == -1) {
Py_DECREF(type_params_name);
return ERROR;
@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e)
return SUCCESS;
}
static const int compare_masks[] = {
[Py_LT] = COMPARISON_LESS_THAN,
[Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
[Py_EQ] = COMPARISON_EQUALS,
[Py_NE] = COMPARISON_NOT_EQUALS,
[Py_GT] = COMPARISON_GREATER_THAN,
[Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
};
static int compiler_addcompare(struct compiler *c, location loc,
cmpop_ty op)
{
@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s)
{
location loc = LOC(s);
expr_ty targ = s->v.AnnAssign.target;
PyObject* mangled;
bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS;
PyObject *mangled;
assert(s->kind == AnnAssign_kind);
@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s)
if (s->v.AnnAssign.simple &&
(c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
c->u->u_scope_type == COMPILER_SCOPE_CLASS)) {
if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) {
VISIT(c, annexpr, s->v.AnnAssign.annotation)
if (future_annotations) {
VISIT(c, annexpr, s->v.AnnAssign.annotation);
ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
ADDOP(c, loc, STORE_SUBSCR);
}
else {
VISIT(c, expr, s->v.AnnAssign.annotation);
if (c->u->u_deferred_annotations == NULL) {
c->u->u_deferred_annotations = PyList_New(0);
if (c->u->u_deferred_annotations == NULL) {
return ERROR;
}
}
PyObject *ptr = PyLong_FromVoidPtr((void *)s);
if (ptr == NULL) {
return ERROR;
}
if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) {
Py_DECREF(ptr);
return ERROR;
}
Py_DECREF(ptr);
}
ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
ADDOP(c, loc, STORE_SUBSCR);
}
break;
case Attribute_kind:
@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
return ERROR;
}
/* Annotation is evaluated last. */
if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
return ERROR;
}
return SUCCESS;

5
Python/executor_cases.c.h generated
View file

@ -4061,6 +4061,11 @@
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
case MAKE_FUNCTION_ANNOTATE:
assert(PyCallable_Check(attr));
assert(func_obj->func_annotate == NULL);
func_obj->func_annotate = attr;
break;
default:
Py_UNREACHABLE();
}

5
Python/generated_cases.c.h generated
View file

@ -5450,6 +5450,11 @@
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
case MAKE_FUNCTION_ANNOTATE:
assert(PyCallable_Check(attr));
assert(func_obj->func_annotate == NULL);
func_obj->func_annotate = attr;
break;
default:
Py_UNREACHABLE();
}

198
Python/symtable.c
View file

@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_varkeywords = 0;
ste->ste_opt_lineno = 0;
ste->ste_opt_col_offset = 0;
ste->ste_annotations_used = 0;
ste->ste_lineno = lineno;
ste->ste_col_offset = col_offset;
ste->ste_end_lineno = end_lineno;
@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_can_see_class_scope = 0;
ste->ste_comp_iter_expr = 0;
ste->ste_needs_classdict = 0;
ste->ste_annotation_block = NULL;
ste->ste_symbols = PyDict_New();
ste->ste_varnames = PyList_New(0);
@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste)
Py_XDECREF(ste->ste_varnames);
Py_XDECREF(ste->ste_children);
Py_XDECREF(ste->ste_directives);
Py_XDECREF(ste->ste_annotation_block);
Py_XDECREF(ste->ste_mangled_names);
PyObject_Free(ste);
}
@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
static int symtable_visit_keyword(struct symtable *st, keyword_ty);
static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key);
static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty,
struct _symtable_entry *parent_ste);
static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key)
return (PySTEntryObject *)v;
}
int
_PySymtable_LookupOptional(struct symtable *st, void *key,
PySTEntryObject **out)
{
PyObject *k = PyLong_FromVoidPtr(key);
if (k == NULL) {
*out = NULL;
return -1;
}
int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out);
Py_DECREF(k);
assert(*out == NULL || PySTEntry_Check(*out));
return result;
}
long
_PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
{
@ -525,6 +544,7 @@ int
_PyST_IsFunctionLike(PySTEntryObject *ste)
{
return ste->ste_type == FunctionBlock
|| ste->ste_type == AnnotationBlock
|| ste->ste_type == TypeVarBoundBlock
|| ste->ste_type == TypeAliasBlock
|| ste->ste_type == TypeParamBlock;
@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st)
}
static int
symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
void *ast, int lineno, int col_offset,
int end_lineno, int end_col_offset)
symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste)
{
PySTEntryObject *prev = NULL, *ste;
ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
if (ste == NULL)
return 0;
if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
Py_DECREF(ste);
return 0;
}
prev = st->st_cur;
PySTEntryObject *prev = st->st_cur;
/* bpo-37757: For now, disallow *all* assignment expressions in the
* outermost iterator expression of a comprehension, even those inside
* a nested comprehension or a lambda expression.
@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
}
/* No need to inherit ste_mangled_names in classes, where all names
* are mangled. */
if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) {
ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
}
/* The entry is owned by the stack. Borrow it for st_cur. */
Py_DECREF(ste);
st->st_cur = ste;
/* Annotation blocks shouldn't have any affect on the symbol table since in
* the compilation stage, they will all be transformed to strings. They are
* only created if future 'annotations' feature is activated. */
if (block == AnnotationBlock) {
/* If "from __future__ import annotations" is active,
* annotation blocks shouldn't have any affect on the symbol table since in
* the compilation stage, they will all be transformed to strings. */
if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) {
return 1;
}
if (block == ModuleBlock)
if (ste->ste_type == ModuleBlock)
st->st_global = st->st_cur->ste_symbols;
if (prev) {
@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
return 1;
}
static int
symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
void *ast, int lineno, int col_offset,
int end_lineno, int end_col_offset)
{
PySTEntryObject *ste = ste_new(st, name, block, ast,
lineno, col_offset, end_lineno, end_col_offset);
if (ste == NULL)
return 0;
int result = symtable_enter_existing_block(st, ste);
Py_DECREF(ste);
return result;
}
static long
symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
{
@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_QUIT(st, 0);
}
switch (s->kind) {
case FunctionDef_kind:
case FunctionDef_kind: {
if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
VISIT_QUIT(st, 0);
if (s->v.FunctionDef.args->defaults)
@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
}
VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params);
}
PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
LOCATION(s));
if (!new_ste) {
VISIT_QUIT(st, 0);
}
if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
s->v.FunctionDef.returns))
s->v.FunctionDef.returns, new_ste)) {
Py_DECREF(new_ste);
VISIT_QUIT(st, 0);
if (!symtable_enter_block(st, s->v.FunctionDef.name,
FunctionBlock, (void *)s,
LOCATION(s)))
}
if (!symtable_enter_existing_block(st, new_ste)) {
Py_DECREF(new_ste);
VISIT_QUIT(st, 0);
}
Py_DECREF(new_ste);
VISIT(st, arguments, s->v.FunctionDef.args);
VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
if (!symtable_exit_block(st))
@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_QUIT(st, 0);
}
break;
}
case ClassDef_kind: {
PyObject *tmp;
if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT(st, expr, s->v.Assign.value);
break;
case AnnAssign_kind:
st->st_cur->ste_annotations_used = 1;
if (s->v.AnnAssign.target->kind == Name_kind) {
expr_ty e_name = s->v.AnnAssign.target;
long cur = symtable_lookup(st, e_name->v.Name.id);
@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
else {
VISIT(st, expr, s->v.AnnAssign.target);
}
if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation,
(void *)((uintptr_t)st->st_cur->ste_id + 1))) {
VISIT_QUIT(st, 0);
}
@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_SEQ(st, withitem, s->v.With.items);
VISIT_SEQ(st, stmt, s->v.With.body);
break;
case AsyncFunctionDef_kind:
case AsyncFunctionDef_kind: {
if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
VISIT_QUIT(st, 0);
if (s->v.AsyncFunctionDef.args->defaults)
@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
}
VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params);
}
PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
LOCATION(s));
if (!new_ste) {
VISIT_QUIT(st, 0);
}
if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
s->v.AsyncFunctionDef.returns))
s->v.AsyncFunctionDef.returns, new_ste))
VISIT_QUIT(st, 0);
if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
FunctionBlock, (void *)s,
s->lineno, s->col_offset,
s->end_lineno, s->end_col_offset))
if (!symtable_enter_existing_block(st, new_ste)) {
Py_DECREF(new_ste);
VISIT_QUIT(st, 0);
}
Py_DECREF(new_ste);
st->st_cur->ste_coroutine = 1;
VISIT(st, arguments, s->v.AsyncFunctionDef.args);
VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_QUIT(st, 0);
}
break;
}
case AsyncWith_kind:
VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
}
static int
symtable_visit_annotation(struct symtable *st, expr_ty annotation)
symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key)
{
int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
if (future_annotations &&
!symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
(void *)annotation, annotation->lineno,
annotation->col_offset, annotation->end_lineno,
annotation->end_col_offset)) {
VISIT_QUIT(st, 0);
struct _symtable_entry *parent_ste = st->st_cur;
if (parent_ste->ste_annotation_block == NULL) {
_Py_block_ty current_type = parent_ste->ste_type;
if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
key, LOCATION(annotation))) {
VISIT_QUIT(st, 0);
}
parent_ste->ste_annotation_block =
(struct _symtable_entry *)Py_NewRef(st->st_cur);
int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
if (current_type == ClassBlock && !future_annotations) {
st->st_cur->ste_can_see_class_scope = 1;
if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) {
return 0;
}
}
_Py_DECLARE_STR(format, ".format");
// The generated __annotate__ function takes a single parameter with the
// internal name ".format".
if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM,
LOCATION(annotation))) {
return 0;
}
if (!symtable_add_def(st, &_Py_STR(format), USE,
LOCATION(annotation))) {
return 0;
}
}
else {
if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) {
VISIT_QUIT(st, 0);
}
}
VISIT(st, expr, annotation);
if (future_annotations && !symtable_exit_block(st)) {
if (!symtable_exit_block(st)) {
VISIT_QUIT(st, 0);
}
return 1;
@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
for (i = 0; i < asdl_seq_LEN(args); i++) {
arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
if (arg->annotation)
if (arg->annotation) {
st->st_cur->ste_annotations_used = 1;
VISIT(st, expr, arg->annotation);
}
}
return 1;
}
static int
symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns,
struct _symtable_entry *function_ste)
{
int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
if (future_annotations &&
!symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
(void *)o, o->lineno, o->col_offset, o->end_lineno,
o->end_col_offset)) {
int is_in_class = st->st_cur->ste_can_see_class_scope;
_Py_block_ty current_type = st->st_cur->ste_type;
if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
(void *)a, LOCATION(o))) {
VISIT_QUIT(st, 0);
}
if (is_in_class || current_type == ClassBlock) {
st->st_cur->ste_can_see_class_scope = 1;
if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) {
return 0;
}
}
_Py_DECLARE_STR(format, ".format");
// We need to insert code that reads this "parameter" to the function.
if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) {
return 0;
}
if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) {
return 0;
}
if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
return 0;
if (a->args && !symtable_visit_argannotations(st, a->args))
return 0;
if (a->vararg && a->vararg->annotation)
if (a->vararg && a->vararg->annotation) {
st->st_cur->ste_annotations_used = 1;
VISIT(st, expr, a->vararg->annotation);
if (a->kwarg && a->kwarg->annotation)
}
if (a->kwarg && a->kwarg->annotation) {
st->st_cur->ste_annotations_used = 1;
VISIT(st, expr, a->kwarg->annotation);
}
if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
return 0;
if (future_annotations && !symtable_exit_block(st)) {
VISIT_QUIT(st, 0);
if (returns) {
st->st_cur->ste_annotations_used = 1;
VISIT(st, expr, returns);
}
if (returns && !symtable_visit_annotation(st, returns)) {
if (!symtable_exit_block(st)) {
VISIT_QUIT(st, 0);
}
return 1;
@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
static int
symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
{
enum _block_type type = st->st_cur->ste_type;
_Py_block_ty type = st->st_cur->ste_type;
if (type == AnnotationBlock)
PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
else if (type == TypeVarBoundBlock)