mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-05 00:11:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Hide list comp variables and support set comprehensions

Browse source
This commit is contained in:
Nick Coghlan 2007-04-15 12:05:43 +00:00
parent 6ef6306dd6
commit 650f0d06d3
29 changed files with 2006 additions and 1323 deletions
Download patch file Download diff file Expand all files Collapse all files

45
Python/Python-ast.c
View file

@ -192,6 +192,11 @@ static char *ListComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *SetComp_type;
static char *SetComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *GeneratorExp_type;
static char *GeneratorExp_fields[]={
"elt",
@ -543,6 +548,8 @@ static int init_types(void)
if (!Set_type) return 0;
ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
if (!ListComp_type) return 0;
SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
if (!SetComp_type) return 0;
GeneratorExp_type = make_type("GeneratorExp", expr_type,
GeneratorExp_fields, 2);
if (!GeneratorExp_type) return 0;
@ -1418,6 +1425,27 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
return p;
}
expr_ty
SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!elt) {
PyErr_SetString(PyExc_ValueError,
"field elt is required for SetComp");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = SetComp_kind;
p->v.SetComp.elt = elt;
p->v.SetComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena)
@ -2416,6 +2444,21 @@ ast2obj_expr(void* _o)
goto failed;
Py_DECREF(value);
break;
case SetComp_kind:
result = PyType_GenericNew(SetComp_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.SetComp.elt);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.SetComp.generators,
ast2obj_comprehension);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
Py_DECREF(value);
break;
case GeneratorExp_kind:
result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
if (!result) goto failed;
@ -3120,6 +3163,8 @@ init_ast(void)
if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return;
if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0)
return;
if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0)
return;
if (PyDict_SetItemString(d, "GeneratorExp",
(PyObject*)GeneratorExp_type) < 0) return;
if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return;

366
Python/ast.c
View file

@ -27,7 +27,6 @@ static stmt_ty ast_for_stmt(struct compiling *, const node *);
static asdl_seq *ast_for_suite(struct compiling *, const node *);
static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
@ -41,6 +40,10 @@ static PyObject *parsestrplus(struct compiling *, const node *n,
#define LINENO(n) ((n)->n_lineno)
#endif
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
static identifier
new_identifier(const char* n, PyArena *arena) {
PyObject* id = PyString_InternFromString(n);
@ -231,7 +234,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
case eval_input: {
expr_ty testlist_ast;
/* XXX Why not gen_for here? */
/* XXX Why not comp_for here? */
testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
if (!testlist_ast)
goto error;
@ -530,19 +533,14 @@ seq_for_testlist(struct compiling *c, const node *n)
asdl_seq *seq;
expr_ty expression;
int i;
assert(TYPE(n) == testlist
|| TYPE(n) == listmaker
|| TYPE(n) == testlist_gexp
|| TYPE(n) == testlist_safe
|| TYPE(n) == testlist1
);
assert(TYPE(n) == testlist || TYPE(n) == testlist_comp);
seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!seq)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test);
assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond);
expression = ast_for_expr(c, CHILD(n, i));
if (!expression)
@ -1022,7 +1020,8 @@ ast_for_funcdef(struct compiling *c, const node *n)
static expr_ty
ast_for_lambdef(struct compiling *c, const node *n)
{
/* lambdef: 'lambda' [varargslist] ':' test */
/* lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
arguments_ty args;
expr_ty expression;
@ -1067,190 +1066,34 @@ ast_for_ifexpr(struct compiling *c, const node *n)
c->c_arena);
}
/* XXX(nnorwitz): the listcomp and genexpr code should be refactored
so there is only a single version. Possibly for loops can also re-use
the code.
*/
/* Count the number of 'for' loop in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_fors(const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_list_for:
n_fors++;
REQ(ch, list_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
else
return n_fors;
count_list_iter:
REQ(ch, list_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == list_for)
goto count_list_for;
else if (TYPE(ch) == list_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_list_iter;
}
else
return n_fors;
}
/* Should never be reached */
PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");
return -1;
}
/* Count the number of 'if' statements in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_ifs(const node *n)
{
int n_ifs = 0;
count_list_iter:
REQ(n, list_iter);
if (TYPE(CHILD(n, 0)) == list_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, list_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
n = CHILD(n, 2);
goto count_list_iter;
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
/* listmaker: test ( list_for | (',' test)* [','] )
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_iter: list_for | list_if
list_if: 'if' test [list_iter]
testlist_safe: test [(',' test)+ [',']]
*/
expr_ty elt;
asdl_seq *listcomps;
int i, n_fors;
node *ch;
REQ(n, listmaker);
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_list_fors(n);
if (n_fors == -1)
return NULL;
listcomps = asdl_seq_new(n_fors, c->c_arena);
if (!listcomps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty lc;
asdl_seq *t;
expr_ty expression;
node *for_ch;
REQ(ch, list_for);
for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
return NULL;
expression = ast_for_testlist(c, CHILD(ch, 3));
if (!expression)
return NULL;
/* Check the # of children rather than the length of t, since
[x for x, in ... ] has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1)
lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL,
c->c_arena);
else
lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!lc)
return NULL;
if (NCH(ch) == 5) {
int j, n_ifs;
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_list_ifs(ch);
if (n_ifs == -1)
return NULL;
ifs = asdl_seq_new(n_ifs, c->c_arena);
if (!ifs)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, list_iter);
ch = CHILD(ch, 0);
REQ(ch, list_if);
asdl_seq_SET(ifs, j, ast_for_expr(c, CHILD(ch, 1)));
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
}
/* on exit, must guarantee that ch is a list_for */
if (TYPE(ch) == list_iter)
ch = CHILD(ch, 0);
lc->ifs = ifs;
}
asdl_seq_SET(listcomps, i, lc);
}
return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/*
Count the number of 'for' loops in a generator expression.
Count the number of 'for' loops in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_fors(const node *n)
count_comp_fors(const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_gen_for:
count_comp_for:
n_fors++;
REQ(ch, gen_for);
REQ(ch, comp_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
else
return n_fors;
count_gen_iter:
REQ(ch, gen_iter);
count_comp_iter:
REQ(ch, comp_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == gen_for)
goto count_gen_for;
else if (TYPE(ch) == gen_if) {
if (TYPE(ch) == comp_for)
goto count_comp_for;
else if (TYPE(ch) == comp_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_gen_iter;
goto count_comp_iter;
}
else
return n_fors;
@ -1258,26 +1101,26 @@ count_gen_fors(const node *n)
/* Should never be reached */
PyErr_SetString(PyExc_SystemError,
"logic error in count_gen_fors");
"logic error in count_comp_fors");
return -1;
}
/* Count the number of 'if' statements in a generator expression.
/* Count the number of 'if' statements in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_ifs(const node *n)
count_comp_ifs(const node *n)
{
int n_ifs = 0;
while (1) {
REQ(n, gen_iter);
if (TYPE(CHILD(n, 0)) == gen_for)
REQ(n, comp_iter);
if (TYPE(CHILD(n, 0)) == comp_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, gen_if);
REQ(n, comp_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
@ -1285,40 +1128,38 @@ count_gen_ifs(const node *n)
}
}
/* TODO(jhylton): Combine with list comprehension code? */
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
ast_for_comprehension(struct compiling *c, const node *n, int type)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] )
argument: [test '='] test [gen_for] # Really [keyword '='] test */
/* testlist_comp: test ( comp_for | (',' test)* [','] )
argument: [test '='] test [comp_for] # Really [keyword '='] test */
expr_ty elt;
asdl_seq *genexps;
asdl_seq *comps;
int i, n_fors;
node *ch;
assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument));
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_gen_fors(n);
n_fors = count_comp_fors(n);
if (n_fors == -1)
return NULL;
genexps = asdl_seq_new(n_fors, c->c_arena);
if (!genexps)
comps = asdl_seq_new(n_fors, c->c_arena);
if (!comps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty ge;
comprehension_ty comp;
asdl_seq *t;
expr_ty expression;
node *for_ch;
REQ(ch, gen_for);
REQ(ch, comp_for);
for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store);
@ -1331,14 +1172,14 @@ ast_for_genexp(struct compiling *c, const node *n)
/* Check the # of children rather than the length of t, since
(x for x, in ...) has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1)
ge = comprehension((expr_ty)asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
comp = comprehension((expr_ty)asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
else
ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
comp = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!ge)
if (!comp)
return NULL;
if (NCH(ch) == 5) {
@ -1346,7 +1187,7 @@ ast_for_genexp(struct compiling *c, const node *n)
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_gen_ifs(ch);
n_ifs = count_comp_ifs(ch);
if (n_ifs == -1)
return NULL;
@ -1355,9 +1196,9 @@ ast_for_genexp(struct compiling *c, const node *n)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, gen_iter);
REQ(ch, comp_iter);
ch = CHILD(ch, 0);
REQ(ch, gen_if);
REQ(ch, comp_if);
expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression)
@ -1366,22 +1207,52 @@ ast_for_genexp(struct compiling *c, const node *n)
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
}
/* on exit, must guarantee that ch is a gen_for */
if (TYPE(ch) == gen_iter)
/* on exit, must guarantee that ch is a comp_for */
if (TYPE(ch) == comp_iter)
ch = CHILD(ch, 0);
ge->ifs = ifs;
comp->ifs = ifs;
}
asdl_seq_SET(genexps, i, ge);
asdl_seq_SET(comps, i, comp);
}
return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_LISTCOMP)
return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
}
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument));
return ast_for_comprehension(c, n, COMP_GENEXP);
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp));
return ast_for_comprehension(c, n, COMP_LISTCOMP);
}
static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_comprehension(c, n, COMP_SETCOMP);
}
static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
/* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']'
| '{' [dictsetmaker] '}' | NAME | NUMBER | STRING+
/* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
| '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
*/
node *ch = CHILD(n, 0);
int bytesmode = 0;
@ -1420,18 +1291,19 @@ ast_for_atom(struct compiling *c, const node *n)
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for))
/* testlist_comp: test ( comp_for | (',' test)* [','] ) */
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for))
return ast_for_genexp(c, ch);
return ast_for_testlist_gexp(c, ch);
return ast_for_testlist(c, ch);
case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, listmaker);
REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts)
@ -1442,27 +1314,32 @@ ast_for_atom(struct compiling *c, const node *n)
else
return ast_for_listcomp(c, ch);
case LBRACE: {
/* dictsetmaker: test ':' test (',' test ':' test)* [','] |
* test (',' test)* [','] */
/* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
* test (gen_for | (',' test)* [',']) */
int i, size;
asdl_seq *keys, *values;
ch = CHILD(n, 1);
if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) {
/* it's a set */
if (TYPE(ch) == RBRACE) {
/* it's an empty dict */
return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
} else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
/* it's a simple set */
size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
keys = asdl_seq_new(size, c->c_arena);
if (!keys)
asdl_seq *elts = asdl_seq_new(size, c->c_arena);
if (!elts)
return NULL;
for (i = 0; i < NCH(ch); i += 2) {
expr_ty expression;
expression = ast_for_expr(c, CHILD(ch, i));
if (!expression)
return NULL;
asdl_seq_SET(keys, i / 2, expression);
asdl_seq_SET(elts, i / 2, expression);
}
return Set(keys, LINENO(n), n->n_col_offset, c->c_arena);
return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
} else if (TYPE(CHILD(ch, 1)) == comp_for) {
/* it's a set comprehension */
return ast_for_setcomp(c, ch);
} else {
/* it's a dict */
size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
@ -1790,6 +1667,7 @@ ast_for_expr(struct compiling *c, const node *n)
{
/* handle the full range of simple expressions
test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
@ -1802,15 +1680,6 @@ ast_for_expr(struct compiling *c, const node *n)
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ('**' factor)*
As well as modified versions that exist for backward compatibility,
to explicitly allow:
[ x for x in lambda: 0, lambda: 1 ]
(which would be ambiguous without these extra rules)
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [vararglist] ':' old_test
*/
asdl_seq *seq;
@ -1819,9 +1688,9 @@ ast_for_expr(struct compiling *c, const node *n)
loop:
switch (TYPE(n)) {
case test:
case old_test:
case test_nocond:
if (TYPE(CHILD(n, 0)) == lambdef ||
TYPE(CHILD(n, 0)) == old_lambdef)
TYPE(CHILD(n, 0)) == lambdef_nocond)
return ast_for_lambdef(c, CHILD(n, 0));
else if (NCH(n) > 1)
return ast_for_ifexpr(c, n);
@ -1947,7 +1816,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
/*
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
| '**' test)
argument: [test '='] test [gen_for] # Really [keyword '='] test
argument: [test '='] test [comp_for] # Really [keyword '='] test
*/
int i, nargs, nkeywords, ngens;
@ -1965,7 +1834,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
if (TYPE(ch) == argument) {
if (NCH(ch) == 1)
nargs++;
else if (TYPE(CHILD(ch, 1)) == gen_for)
else if (TYPE(CHILD(ch, 1)) == comp_for)
ngens++;
else
nkeywords++;
@ -2005,7 +1874,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
return NULL;
asdl_seq_SET(args, nargs++, e);
}
else if (TYPE(CHILD(ch, 1)) == gen_for) {
else if (TYPE(CHILD(ch, 1)) == comp_for) {
e = ast_for_genexp(c, ch);
if (!e)
return NULL;
@ -2057,18 +1926,16 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
static expr_ty
ast_for_testlist(struct compiling *c, const node* n)
{
/* testlist_gexp: test (',' test)* [','] */
/* testlist_comp: test (comp_for | (',' test)* [',']) */
/* testlist: test (',' test)* [','] */
/* testlist_safe: test (',' test)+ [','] */
/* testlist1: test (',' test)* */
assert(NCH(n) > 0);
if (TYPE(n) == testlist_gexp) {
if (TYPE(n) == testlist_comp) {
if (NCH(n) > 1)
assert(TYPE(CHILD(n, 1)) != gen_for);
assert(TYPE(CHILD(n, 1)) != comp_for);
}
else {
assert(TYPE(n) == testlist ||
TYPE(n) == testlist_safe ||
TYPE(n) == testlist1);
}
if (NCH(n) == 1)
@ -2081,17 +1948,6 @@ ast_for_testlist(struct compiling *c, const node* n)
}
}
static expr_ty
ast_for_testlist_gexp(struct compiling *c, const node* n)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] ) */
/* argument: test [ gen_for ] */
assert(TYPE(n) == testlist_gexp || TYPE(n) == argument);
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for)
return ast_for_genexp(c, n);
return ast_for_testlist(c, n);
}
static stmt_ty
ast_for_expr_stmt(struct compiling *c, const node *n)
{

12
Python/ceval.c
View file

@ -1241,6 +1241,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
}
break;
case SET_ADD:
w = POP();
v = POP();
err = PySet_Add(v, w);
Py_DECREF(v);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
continue;
}
break;
case INPLACE_POWER:
w = POP();
v = TOP();

295
Python/compile.c
View file

@ -39,6 +39,10 @@ int Py_OptimizeFlag = 0;
#define DEFAULT_CODE_SIZE 128
#define DEFAULT_LNOTAB_SIZE 16
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
struct instr {
unsigned i_jabs : 1;
unsigned i_jrel : 1;
@ -360,7 +364,7 @@ dictbytype(PyObject *src, int scope_type, int flag, int offset)
while (PyDict_Next(src, &pos, &k, &v)) {
/* XXX this should probably be a macro in symtable.h */
assert(PyInt_Check(v));
scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK;
scope = (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
if (scope == scope_type || PyInt_AS_LONG(v) & flag) {
PyObject *tuple, *item = PyInt_FromLong(i);
@ -673,6 +677,7 @@ opcode_stack_effect(int opcode, int oparg)
case UNARY_INVERT:
return 0;
case SET_ADD:
case LIST_APPEND:
return -2;
@ -2724,15 +2729,31 @@ compiler_call_helper(struct compiler *c,
return 1;
}
/* List and set comprehensions and generator expressions work by creating a
nested function to perform the actual iteration. This means that the
iteration variables don't leak into the current scope.
The defined function is called immediately following its definition, with the
result of that call being the result of the expression.
The LC/SC version returns the populated container, while the GE version is
flagged in symtable.c as a generator, so it returns the generator object
when the function is called.
This code *knows* that the loop cannot contain break, continue, or return,
so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
Possible cleanups:
- iterate over the generator sequence instead of using recursion
*/
static int
compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
asdl_seq *generators, int gen_index,
expr_ty elt)
compiler_comprehension_generator(struct compiler *c, PyObject *tmpname,
asdl_seq *generators, int gen_index,
expr_ty elt, int type)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty l;
comprehension_ty gen;
basicblock *start, *anchor, *skip, *if_cleanup;
int i, n;
@ -2742,104 +2763,11 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
anchor = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL)
return 0;
l = (comprehension_ty)asdl_seq_GET(generators, gen_index);
VISIT(c, expr, l->iter);
ADDOP(c, GET_ITER);
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, l->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(l->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i);
VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
ADDOP(c, POP_TOP);
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_listcomp_generator(c, tmpname,
generators, gen_index, elt))
anchor == NULL)
return 0;
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
compiler_use_next_block(c, skip);
}
for (i = 0; i < n; i++) {
ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0)
compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP);
}
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
/* delete the temporary list name added to locals */
if (gen_index == 1)
if (!compiler_nameop(c, tmpname, Del))
return 0;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
return 1;
}
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
identifier tmp;
int rc = 0;
asdl_seq *generators = e->v.ListComp.generators;
assert(e->kind == ListComp_kind);
tmp = compiler_new_tmpname(c);
if (!tmp)
return 0;
ADDOP_I(c, BUILD_LIST, 0);
ADDOP(c, DUP_TOP);
if (compiler_nameop(c, tmp, Store))
rc = compiler_listcomp_generator(c, tmp, generators, 0,
e->v.ListComp.elt);
Py_DECREF(tmp);
return rc;
}
static int
compiler_genexp_generator(struct compiler *c,
asdl_seq *generators, int gen_index,
expr_ty elt)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty ge;
basicblock *start, *anchor, *skip, *if_cleanup, *end;
int i, n;
start = compiler_new_block(c);
skip = compiler_new_block(c);
if_cleanup = compiler_new_block(c);
anchor = compiler_new_block(c);
end = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL || end == NULL)
return 0;
ge = (comprehension_ty)asdl_seq_GET(generators, gen_index);
ADDOP_JREL(c, SETUP_LOOP, end);
if (!compiler_push_fblock(c, LOOP, start))
return 0;
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1;
@ -2847,18 +2775,18 @@ compiler_genexp_generator(struct compiler *c,
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, ge->iter);
VISIT(c, expr, gen->iter);
ADDOP(c, GET_ITER);
}
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, ge->target);
VISIT(c, expr, gen->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(ge->ifs);
n = asdl_seq_LEN(gen->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i);
expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
@ -2866,14 +2794,35 @@ compiler_genexp_generator(struct compiler *c,
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_genexp_generator(c, generators, gen_index, elt))
return 0;
if (!compiler_comprehension_generator(c, tmpname,
generators, gen_index,
elt, type))
return 0;
/* only append after the last 'for' generator */
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
/* comprehension specific code */
switch (type) {
case COMP_GENEXP:
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
break;
case COMP_LISTCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
break;
case COMP_SETCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, SET_ADD);
break;
default:
return 0;
}
compiler_use_next_block(c, skip);
}
@ -2881,52 +2830,116 @@ compiler_genexp_generator(struct compiler *c,
ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0)
compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP);
}
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, LOOP, start);
compiler_use_next_block(c, end);
return 1;
}
static int
compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
asdl_seq *generators, expr_ty elt)
{
PyCodeObject *co = NULL;
identifier tmp = NULL;
expr_ty outermost_iter;
outermost_iter = ((comprehension_ty)
asdl_seq_GET(generators, 0))->iter;
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
goto error;
if (type != COMP_GENEXP) {
tmp = compiler_new_tmpname(c);
if (!tmp)
goto error_in_scope;
ADDOP_I(c, (type == COMP_LISTCOMP ?
BUILD_LIST : BUILD_SET), 0);
ADDOP(c, DUP_TOP);
if (!compiler_nameop(c, tmp, Store))
goto error_in_scope;
}
if (!compiler_comprehension_generator(c, tmp, generators, 0, elt, type))
goto error_in_scope;
if (type != COMP_GENEXP) {
ADDOP(c, RETURN_VALUE);
}
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
goto error;
if (!compiler_make_closure(c, co, 0))
goto error;
Py_DECREF(co);
Py_XDECREF(tmp);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
error_in_scope:
compiler_exit_scope(c);
error:
Py_XDECREF(co);
Py_XDECREF(tmp);
return 0;
}
static int
compiler_genexp(struct compiler *c, expr_ty e)
{
static identifier name;
PyCodeObject *co;
expr_ty outermost_iter = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators,
0)))->iter;
if (!name) {
name = PyString_FromString("<genexpr>");
name = PyString_FromString("<genexp>");
if (!name)
return 0;
}
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
return 0;
compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0,
e->v.GeneratorExp.elt);
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
return 0;
compiler_make_closure(c, co, 0);
Py_DECREF(co);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
assert(e->kind == GeneratorExp_kind);
return compiler_comprehension(c, e, COMP_GENEXP, name,
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt);
}
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<listcomp>");
if (!name)
return 0;
}
assert(e->kind == ListComp_kind);
return compiler_comprehension(c, e, COMP_LISTCOMP, name,
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
compiler_setcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<setcomp>");
if (!name)
return 0;
}
assert(e->kind == SetComp_kind);
return compiler_comprehension(c, e, COMP_SETCOMP, name,
e->v.SetComp.generators,
e->v.SetComp.elt);
}
static int
compiler_visit_keyword(struct compiler *c, keyword_ty k)
{
@ -3145,10 +3158,12 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
VISIT_SEQ(c, expr, e->v.Set.elts);
ADDOP_I(c, BUILD_SET, n);
break;
case ListComp_kind:
return compiler_listcomp(c, e);
case GeneratorExp_kind:
return compiler_genexp(c, e);
case ListComp_kind:
return compiler_listcomp(c, e);
case SetComp_kind:
return compiler_setcomp(c, e);
case Yield_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function");

986
Python/graminit.c
View file

File diff suppressed because it is too large Load diff

347
Python/symtable.c
View file

@ -76,7 +76,7 @@ PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_generator = 0;
ste->ste_returns_value = 0;
if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0)
if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
goto fail;
return ste;
@ -172,6 +172,8 @@ static int symtable_exit_block(struct symtable *st, void *ast);
static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
static int symtable_visit_expr(struct symtable *st, expr_ty s);
static int symtable_visit_genexp(struct symtable *st, expr_ty s);
static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
static int symtable_visit_arguments(struct symtable *st, arguments_ty);
static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
static int symtable_visit_alias(struct symtable *st, alias_ty);
@ -186,7 +188,8 @@ static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
static identifier top = NULL, lambda = NULL, genexpr = NULL;
static identifier top = NULL, lambda = NULL, genexpr = NULL,
listcomp = NULL, setcomp = NULL;
#define GET_IDENTIFIER(VAR) \
((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR)))
@ -204,14 +207,13 @@ symtable_new(void)
return NULL;
st->st_filename = NULL;
st->st_symbols = NULL;
st->st_blocks = NULL;
if ((st->st_stack = PyList_New(0)) == NULL)
goto fail;
if ((st->st_symbols = PyDict_New()) == NULL)
if ((st->st_blocks = PyDict_New()) == NULL)
goto fail;
st->st_cur = NULL;
st->st_tmpname = 0;
st->st_private = NULL;
return st;
fail:
@ -230,6 +232,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
return st;
st->st_filename = filename;
st->st_future = future;
/* Make the initial symbol information gathering pass */
if (!GET_IDENTIFIER(top) ||
!symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) {
PySymtable_Free(st);
@ -238,7 +241,6 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
st->st_top = st->st_cur;
st->st_cur->ste_unoptimized = OPT_TOPLEVEL;
/* Any other top-level initialization? */
switch (mod->kind) {
case Module_kind:
seq = mod->v.Module.body;
@ -267,6 +269,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
PySymtable_Free(st);
return NULL;
}
/* Make the second symbol analysis pass */
if (symtable_analyze(st))
return st;
PySymtable_Free(st);
@ -280,7 +283,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
void
PySymtable_Free(struct symtable *st)
{
Py_XDECREF(st->st_symbols);
Py_XDECREF(st->st_blocks);
Py_XDECREF(st->st_stack);
PyMem_Free((void *)st);
}
@ -293,7 +296,7 @@ PySymtable_Lookup(struct symtable *st, void *key)
k = PyLong_FromVoidPtr(key);
if (k == NULL)
return NULL;
v = PyDict_GetItem(st->st_symbols, k);
v = PyDict_GetItem(st->st_blocks, k);
if (v) {
assert(PySTEntry_Check(v));
Py_INCREF(v);
@ -314,7 +317,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
if (!v)
return 0;
assert(PyInt_Check(v));
return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK;
return (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
}
@ -325,7 +328,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
it determines which local variables are cell variables; they provide
bindings that are used for free variables in enclosed blocks.
There are also two kinds of free variables, implicit and explicit. An
There are also two kinds of global variables, implicit and explicit. An
explicit global is declared with the global statement. An implicit
global is a free variable for which the compiler has found no binding
in an enclosing function scope. The implicit global is either a global
@ -337,24 +340,30 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
TODO(jhylton): Discuss nonlocal
The symbol table requires two passes to determine the scope of each name.
The first pass collects raw facts from the AST: the name is a parameter
here, the name is used by not defined here, etc. The second pass analyzes
these facts during a pass over the PySTEntryObjects created during pass 1.
The first pass collects raw facts from the AST via the symtable_visit_*
functions: the name is a parameter here, the name is used but not defined
here, etc. The second pass analyzes these facts during a pass over the
PySTEntryObjects created during pass 1.
When a function is entered during the second pass, the parent passes
the set of all name bindings visible to its children. These bindings
are used to determine if the variable is free or an implicit global.
are used to determine if non-local variables are free or implicit globals.
After doing the local analysis, it analyzes each of its child blocks
using an updated set of name bindings.
using an updated set of name bindings.
The children update the free variable set. If a local variable is free
in a child, the variable is marked as a cell. The current function must
provide runtime storage for the variable that may outlive the function's
frame. Cell variables are removed from the free set before the analyze
function returns to its parent.
The children update the free variable set. If a local variable is added to
the free variable set by the child, the variable is marked as a cell. The
function object being defined must provide runtime storage for the variable
that may outlive the function's frame. Cell variables are removed from the
free set before the analyze function returns to its parent.
The sets of bound and free variables are implemented as dictionaries
mapping strings to None.
During analysis, the names are:
symbols: dict mapping from symbol names to flag values (including offset scope values)
scopes: dict mapping from symbol names to scope values (no offset)
local: set of all symbol names local to the current scope
bound: set of all symbol names local to a containing function scope
free: set of all symbol names referenced but not bound in child scopes
global: set of all symbol names explicitly declared as global
*/
#define SET_SCOPE(DICT, NAME, I) { \
@ -375,14 +384,14 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
*/
static int
analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
PyObject *bound, PyObject *local, PyObject *free,
PyObject *global)
{
if (flags & DEF_GLOBAL) {
if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and global",
"name '%s' is parameter and global",
PyString_AS_STRING(name));
return 0;
}
@ -392,41 +401,37 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
PyString_AS_STRING(name));
return 0;
}
SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
if (PyDict_SetItem(global, name, Py_None) < 0)
SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
if (PySet_Add(global, name) < 0)
return 0;
if (bound && (PySet_Discard(bound, name) < 0))
return 0;
if (bound && PyDict_GetItem(bound, name)) {
if (PyDict_DelItem(bound, name) < 0)
return 0;
}
return 1;
}
if (flags & DEF_NONLOCAL) {
if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and nonlocal",
"name '%s' is parameter and nonlocal",
PyString_AS_STRING(name));
return 0;
}
if (!PyDict_GetItem(bound, name)) {
if (!PySet_Contains(bound, name)) {
PyErr_Format(PyExc_SyntaxError,
"no binding for nonlocal '%s' found",
PyString_AS_STRING(name));
return 0;
}
SET_SCOPE(dict, name, FREE);
SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0;
return PySet_Add(free, name) >= 0;
}
if (flags & DEF_BOUND) {
SET_SCOPE(dict, name, LOCAL);
if (PyDict_SetItem(local, name, Py_None) < 0)
SET_SCOPE(scopes, name, LOCAL);
if (PySet_Add(local, name) < 0)
return 0;
if (PySet_Discard(global, name) < 0)
return 0;
if (PyDict_GetItem(global, name)) {
if (PyDict_DelItem(global, name) < 0)
return 0;
}
return 1;
}
/* If an enclosing block has a binding for this name, it
@ -434,21 +439,21 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
Note that having a non-NULL bound implies that the block
is nested.
*/
if (bound && PyDict_GetItem(bound, name)) {
SET_SCOPE(dict, name, FREE);
if (bound && PySet_Contains(bound, name)) {
SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0;
return PySet_Add(free, name) >= 0;
}
/* If a parent has a global statement, then call it global
explicit? It could also be global implicit.
*/
if (global && PyDict_GetItem(global, name)) {
SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
if (global && PySet_Contains(global, name)) {
SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
return 1;
}
if (ste->ste_nested)
ste->ste_free = 1;
SET_SCOPE(dict, name, GLOBAL_IMPLICIT);
SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
return 1;
}
@ -463,35 +468,35 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
*/
static int
analyze_cells(PyObject *scope, PyObject *free)
analyze_cells(PyObject *scopes, PyObject *free)
{
PyObject *name, *v, *w;
PyObject *name, *v, *v_cell;
int success = 0;
Py_ssize_t pos = 0;
w = PyInt_FromLong(CELL);
if (!w)
v_cell = PyInt_FromLong(CELL);
if (!v_cell)
return 0;
while (PyDict_Next(scope, &pos, &name, &v)) {
long flags;
while (PyDict_Next(scopes, &pos, &name, &v)) {
long scope;
assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v);
if (flags != LOCAL)
scope = PyInt_AS_LONG(v);
if (scope != LOCAL)
continue;
if (!PyDict_GetItem(free, name))
if (!PySet_Contains(free, name))
continue;
/* Replace LOCAL with CELL for this name, and remove
from free. It is safe to replace the value of name
in the dict, because it will not cause a resize.
*/
if (PyDict_SetItem(scope, name, w) < 0)
if (PyDict_SetItem(scopes, name, v_cell) < 0)
goto error;
if (!PyDict_DelItem(free, name) < 0)
if (PySet_Discard(free, name) < 0)
goto error;
}
success = 1;
error:
Py_DECREF(w);
Py_DECREF(v_cell);
return success;
}
@ -526,77 +531,91 @@ check_unoptimized(const PySTEntryObject* ste) {
return 0;
}
/* Enter the final scope information into the st_symbols dict.
/* Enter the final scope information into the ste_symbols dict.
*
* All arguments are dicts. Modifies symbols, others are read-only.
*/
static int
update_symbols(PyObject *symbols, PyObject *scope,
update_symbols(PyObject *symbols, PyObject *scopes,
PyObject *bound, PyObject *free, int classflag)
{
PyObject *name, *v, *u, *w, *free_value = NULL;
PyObject *name = NULL, *itr = NULL;
PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
Py_ssize_t pos = 0;
/* Update scope information for all symbols in this scope */
while (PyDict_Next(symbols, &pos, &name, &v)) {
long i, flags;
long scope, flags;
assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v);
w = PyDict_GetItem(scope, name);
assert(w && PyInt_Check(w));
i = PyInt_AS_LONG(w);
flags |= (i << SCOPE_OFF);
u = PyInt_FromLong(flags);
if (!u)
v_scope = PyDict_GetItem(scopes, name);
assert(v_scope && PyInt_Check(v_scope));
scope = PyInt_AS_LONG(v_scope);
flags |= (scope << SCOPE_OFFSET);
v_new = PyInt_FromLong(flags);
if (!v_new)
return 0;
if (PyDict_SetItem(symbols, name, u) < 0) {
Py_DECREF(u);
if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(v_new);
return 0;
}
Py_DECREF(u);
Py_DECREF(v_new);
}
free_value = PyInt_FromLong(FREE << SCOPE_OFF);
if (!free_value)
/* Record not yet resolved free variables from children (if any) */
v_free = PyInt_FromLong(FREE << SCOPE_OFFSET);
if (!v_free)
return 0;
/* add a free variable when it's only use is for creating a closure */
pos = 0;
while (PyDict_Next(free, &pos, &name, &v)) {
PyObject *o = PyDict_GetItem(symbols, name);
itr = PyObject_GetIter(free);
if (!itr)
goto error;
if (o) {
/* It could be a free variable in a method of
while ((name = PyIter_Next(itr))) {
v = PyDict_GetItem(symbols, name);
/* Handle symbol that already exists in this scope */
if (v) {
/* Handle a free variable in a method of
the class that has the same name as a local
or global in the class scope.
*/
if (classflag &&
PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) {
long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS;
o = PyInt_FromLong(i);
if (!o) {
Py_DECREF(free_value);
return 0;
PyInt_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
long flags = PyInt_AS_LONG(v) | DEF_FREE_CLASS;
v_new = PyInt_FromLong(flags);
if (!v_new) {
goto error;
}
if (PyDict_SetItem(symbols, name, o) < 0) {
Py_DECREF(o);
Py_DECREF(free_value);
return 0;
if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(v_new);
goto error;
}
Py_DECREF(o);
Py_DECREF(v_new);
}
/* else it's not free, probably a cell */
/* It's a cell, or already a free variable in this scope */
Py_DECREF(name);
continue;
}
if (!PyDict_GetItem(bound, name))
/* Handle global symbol */
if (!PySet_Contains(bound, name)) {
Py_DECREF(name);
continue; /* it's a global */
if (PyDict_SetItem(symbols, name, free_value) < 0) {
Py_DECREF(free_value);
return 0;
}
/* Propagate new free symbol up the lexical stack */
if (PyDict_SetItem(symbols, name, v_free) < 0) {
goto error;
}
Py_DECREF(name);
}
Py_DECREF(free_value);
Py_DECREF(itr);
Py_DECREF(v_free);
return 1;
error:
Py_XDECREF(v_free);
Py_XDECREF(itr);
Py_XDECREF(name);
return 0;
}
/* Make final symbol table decisions for block of ste.
@ -611,59 +630,74 @@ static int
analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
PyObject *global)
{
PyObject *name, *v, *local = NULL, *scope = NULL, *newbound = NULL;
PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
PyObject *newglobal = NULL, *newfree = NULL;
int i, success = 0;
Py_ssize_t pos = 0;
local = PyDict_New();
scopes = PyDict_New();
if (!scopes)
goto error;
local = PySet_New(NULL);
if (!local)
goto error;
scope = PyDict_New();
if (!scope)
goto error;
newglobal = PyDict_New();
newglobal = PySet_New(NULL);
if (!newglobal)
goto error;
newfree = PyDict_New();
newfree = PySet_New(NULL);
if (!newfree)
goto error;
newbound = PyDict_New();
newbound = PySet_New(NULL);
if (!newbound)
goto error;
/* Class namespace has no effect on names visible in
nested functions, so populate the global and bound
sets to be passed to child blocks before analyzing
this one.
*/
if (ste->ste_type == ClassBlock) {
/* make a copy of globals before calling analyze_name(),
because global statements in the class have no effect
on nested functions.
*/
if (PyDict_Update(newglobal, global) < 0)
goto error;
if (bound)
if (PyDict_Update(newbound, bound) < 0)
/* Pass down previously bound symbols */
if (bound) {
if (!PyNumber_InPlaceOr(newbound, bound))
goto error;
Py_DECREF(newbound);
}
/* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error;
Py_DECREF(newglobal);
}
/* Analyze symbols in current scope */
assert(PySTEntry_Check(ste));
assert(PyDict_Check(ste->ste_symbols));
while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
long flags = PyInt_AS_LONG(v);
if (!analyze_name(ste, scope, name, flags, bound, local, free,
if (!analyze_name(ste, scopes, name, flags, bound, local, free,
global))
goto error;
}
/* Populate global and bound sets to be passed to children.
*/
if (ste->ste_type != ClassBlock) {
/* Add function locals to bound set */
if (ste->ste_type == FunctionBlock) {
if (PyDict_Update(newbound, local) < 0)
if (!PyNumber_InPlaceOr(newbound, local))
goto error;
Py_DECREF(newbound);
}
/* Pass down previously bound symbols */
if (bound) {
if (PyDict_Update(newbound, bound) < 0)
if (!PyNumber_InPlaceOr(newbound, bound))
goto error;
Py_DECREF(newbound);
}
if (PyDict_Update(newglobal, global) < 0)
/* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error;
Py_DECREF(newglobal);
}
/* Recursively call analyze_block() on each child block */
@ -674,24 +708,28 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
entry = (PySTEntryObject*)c;
if (!analyze_block(entry, newbound, newfree, newglobal))
goto error;
/* Check if any children have free variables */
if (entry->ste_free || entry->ste_child_free)
ste->ste_child_free = 1;
}
if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree))
/* Check if any local variables need to be converted to cell variables */
if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
goto error;
if (!update_symbols(ste->ste_symbols, scope, bound, newfree,
/* Records the results of the analysis in the symbol table entry */
if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
ste->ste_type == ClassBlock))
goto error;
if (!check_unoptimized(ste))
goto error;
if (PyDict_Update(free, newfree) < 0)
if (!PyNumber_InPlaceOr(free, newfree))
goto error;
Py_DECREF(free);
success = 1;
error:
Py_XDECREF(scopes);
Py_XDECREF(local);
Py_XDECREF(scope);
Py_XDECREF(newbound);
Py_XDECREF(newglobal);
Py_XDECREF(newfree);
@ -706,10 +744,10 @@ symtable_analyze(struct symtable *st)
PyObject *free, *global;
int r;
free = PyDict_New();
free = PySet_New(NULL);
if (!free)
return 0;
global = PyDict_New();
global = PySet_New(NULL);
if (!global) {
Py_DECREF(free);
return 0;
@ -1200,16 +1238,18 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
case Set_kind:
VISIT_SEQ(st, expr, e->v.Set.elts);
break;
case ListComp_kind:
if (!symtable_new_tmpname(st))
return 0;
VISIT(st, expr, e->v.ListComp.elt);
VISIT_SEQ(st, comprehension, e->v.ListComp.generators);
break;
case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e))
return 0;
break;
case ListComp_kind:
if (!symtable_visit_listcomp(st, e))
return 0;
break;
case SetComp_kind:
if (!symtable_visit_setcomp(st, e))
return 0;
break;
case Yield_kind:
if (e->v.Yield.value)
VISIT(st, expr, e->v.Yield.value);
@ -1479,27 +1519,60 @@ symtable_visit_slice(struct symtable *st, slice_ty s)
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
symtable_handle_comprehension(struct symtable *st, expr_ty e,
identifier scope_name,
asdl_seq *generators, expr_ty elt)
{
int is_generator = (e->kind == GeneratorExp_kind);
int needs_tmp = !is_generator;
comprehension_ty outermost = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators, 0)));
asdl_seq_GET(generators, 0));
/* Outermost iterator is evaluated in current scope */
VISIT(st, expr, outermost->iter);
/* Create generator scope for the rest */
if (!GET_IDENTIFIER(genexpr) ||
!symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, 0)) {
/* Create comprehension scope for the rest */
if (!scope_name ||
!symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
return 0;
}
st->st_cur->ste_generator = 1;
st->st_cur->ste_generator = is_generator;
/* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) {
symtable_exit_block(st, (void *)e);
return 0;
}
/* Allocate temporary name if needed */
if (needs_tmp && !symtable_new_tmpname(st)) {
symtable_exit_block(st, (void *)e);
return 0;
}
VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e);
VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e);
VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension,
e->v.GeneratorExp.generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e);
generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, elt, (void*)e);
return symtable_exit_block(st, (void *)e);
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt);
}
static int
symtable_visit_listcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp),
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
symtable_visit_setcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
e->v.SetComp.generators,
e->v.SetComp.elt);
}