mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-28 05:34:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #2333: Backport set and dict comprehensions syntax.

Browse source
This commit is contained in:
Alexandre Vassalotti 2010-01-11 22:36:12 +00:00
parent 0ca7452794
commit b646547bb4
26 changed files with 1296 additions and 352 deletions
Download patch file Download diff file Expand all files Collapse all files

214
Python/Python-ast.c
View file

@ -197,6 +197,17 @@ static char *ListComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *SetComp_type;
static char *SetComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *DictComp_type;
static char *DictComp_fields[]={
"key",
"value",
"generators",
};
static PyTypeObject *GeneratorExp_type;
static char *GeneratorExp_fields[]={
"elt",
@ -726,6 +737,10 @@ static int init_types(void)
if (!Set_type) return 0;
ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
if (!ListComp_type) return 0;
SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
if (!SetComp_type) return 0;
DictComp_type = make_type("DictComp", expr_type, DictComp_fields, 3);
if (!DictComp_type) return 0;
GeneratorExp_type = make_type("GeneratorExp", expr_type,
GeneratorExp_fields, 2);
if (!GeneratorExp_type) return 0;
@ -1630,6 +1645,54 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
return p;
}
expr_ty
SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!elt) {
PyErr_SetString(PyExc_ValueError,
"field elt is required for SetComp");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = SetComp_kind;
p->v.SetComp.elt = elt;
p->v.SetComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena)
{
expr_ty p;
if (!key) {
PyErr_SetString(PyExc_ValueError,
"field key is required for DictComp");
return NULL;
}
if (!value) {
PyErr_SetString(PyExc_ValueError,
"field value is required for DictComp");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = DictComp_kind;
p->v.DictComp.key = key;
p->v.DictComp.value = value;
p->v.DictComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena)
@ -2610,6 +2673,41 @@ ast2obj_expr(void* _o)
goto failed;
Py_DECREF(value);
break;
case SetComp_kind:
result = PyType_GenericNew(SetComp_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.SetComp.elt);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.SetComp.generators,
ast2obj_comprehension);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
Py_DECREF(value);
break;
case DictComp_kind:
result = PyType_GenericNew(DictComp_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.DictComp.key);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "key", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_expr(o->v.DictComp.value);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.DictComp.generators,
ast2obj_comprehension);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
Py_DECREF(value);
break;
case GeneratorExp_kind:
result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
if (!result) goto failed;
@ -4974,6 +5072,118 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)SetComp_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
expr_ty elt;
asdl_seq* generators;
if (PyObject_HasAttrString(obj, "elt")) {
int res;
tmp = PyObject_GetAttrString(obj, "elt");
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &elt, arena);
if (res != 0) goto failed;
Py_XDECREF(tmp);
tmp = NULL;
} else {
PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp");
return 1;
}
if (PyObject_HasAttrString(obj, "generators")) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = PyObject_GetAttrString(obj, "generators");
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
generators = asdl_seq_new(len, arena);
if (generators == NULL) goto failed;
for (i = 0; i < len; i++) {
comprehension_ty value;
res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(generators, i, value);
}
Py_XDECREF(tmp);
tmp = NULL;
} else {
PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from SetComp");
return 1;
}
*out = SetComp(elt, generators, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)DictComp_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
expr_ty key;
expr_ty value;
asdl_seq* generators;
if (PyObject_HasAttrString(obj, "key")) {
int res;
tmp = PyObject_GetAttrString(obj, "key");
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &key, arena);
if (res != 0) goto failed;
Py_XDECREF(tmp);
tmp = NULL;
} else {
PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp");
return 1;
}
if (PyObject_HasAttrString(obj, "value")) {
int res;
tmp = PyObject_GetAttrString(obj, "value");
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &value, arena);
if (res != 0) goto failed;
Py_XDECREF(tmp);
tmp = NULL;
} else {
PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp");
return 1;
}
if (PyObject_HasAttrString(obj, "generators")) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = PyObject_GetAttrString(obj, "generators");
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
generators = asdl_seq_new(len, arena);
if (generators == NULL) goto failed;
for (i = 0; i < len; i++) {
comprehension_ty value;
res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(generators, i, value);
}
Py_XDECREF(tmp);
tmp = NULL;
} else {
PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from DictComp");
return 1;
}
*out = DictComp(key, value, generators, lineno, col_offset,
arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)GeneratorExp_type);
if (isinstance == -1) {
return 1;
@ -6419,6 +6629,10 @@ init_ast(void)
if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return;
if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0)
return;
if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0)
return;
if (PyDict_SetItemString(d, "DictComp", (PyObject*)DictComp_type) < 0)
return;
if (PyDict_SetItemString(d, "GeneratorExp",
(PyObject*)GeneratorExp_type) < 0) return;
if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return;

235
Python/ast.c
View file

@ -31,7 +31,7 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);
static expr_ty ast_for_testlist_comp(struct compiling *, const node *);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
@ -44,6 +44,9 @@ static PyObject *parsestrplus(struct compiling *, const node *n);
#define LINENO(n) ((n)->n_lineno)
#endif
#define COMP_GENEXP 0
#define COMP_SETCOMP 1
static identifier
new_identifier(const char* n, PyArena *arena) {
PyObject* id = PyString_InternFromString(n);
@ -268,7 +271,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
case eval_input: {
expr_ty testlist_ast;
/* XXX Why not gen_for here? */
/* XXX Why not comp_for here? */
testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
if (!testlist_ast)
goto error;
@ -430,6 +433,12 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
case ListComp_kind:
expr_name = "list comprehension";
break;
case SetComp_kind:
expr_name = "set comprehension";
break;
case DictComp_kind:
expr_name = "dict comprehension";
break;
case Dict_kind:
case Num_kind:
case Str_kind:
@ -573,7 +582,7 @@ seq_for_testlist(struct compiling *c, const node *n)
int i;
assert(TYPE(n) == testlist ||
TYPE(n) == listmaker ||
TYPE(n) == testlist_gexp ||
TYPE(n) == testlist_comp ||
TYPE(n) == testlist_safe ||
TYPE(n) == testlist1);
@ -1150,60 +1159,60 @@ ast_for_listcomp(struct compiling *c, const node *n)
return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/* Count the number of 'for' loops in a generator expression.
/*
Count the number of 'for' loops in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_fors(struct compiling *c, const node *n)
count_comp_fors(struct compiling *c, const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_gen_for:
count_comp_for:
n_fors++;
REQ(ch, gen_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
REQ(n, comp_for);
if (NCH(n) == 5)
n = CHILD(n, 4);
else
return n_fors;
count_gen_iter:
REQ(ch, gen_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == gen_for)
goto count_gen_for;
else if (TYPE(ch) == gen_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_gen_iter;
count_comp_iter:
REQ(n, comp_iter);
n = CHILD(n, 0);
if (TYPE(n) == comp_for)
goto count_comp_for;
else if (TYPE(n) == comp_if) {
if (NCH(n) == 3) {
n = CHILD(n, 2);
goto count_comp_iter;
}
else
return n_fors;
}
/* Should never be reached */
PyErr_SetString(PyExc_SystemError,
"logic error in count_gen_fors");
"logic error in count_comp_fors");
return -1;
}
/* Count the number of 'if' statements in a generator expression.
/* Count the number of 'if' statements in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_ifs(struct compiling *c, const node *n)
count_comp_ifs(struct compiling *c, const node *n)
{
int n_ifs = 0;
while (1) {
REQ(n, gen_iter);
if (TYPE(CHILD(n, 0)) == gen_for)
REQ(n, comp_iter);
if (TYPE(CHILD(n, 0)) == comp_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, gen_if);
REQ(n, comp_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
@ -1211,46 +1220,33 @@ count_gen_ifs(struct compiling *c, const node *n)
}
}
/* TODO(jhylton): Combine with list comprehension code? */
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
static asdl_seq *
ast_for_comprehension(struct compiling *c, const node *n)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] )
argument: [test '='] test [gen_for] # Really [keyword '='] test */
expr_ty elt;
asdl_seq *genexps;
int i, n_fors;
node *ch;
assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument));
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_gen_fors(c, n);
asdl_seq *comps;
n_fors = count_comp_fors(c, n);
if (n_fors == -1)
return NULL;
genexps = asdl_seq_new(n_fors, c->c_arena);
if (!genexps)
comps = asdl_seq_new(n_fors, c->c_arena);
if (!comps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty ge;
comprehension_ty comp;
asdl_seq *t;
expr_ty expression, first;
node *for_ch;
REQ(ch, gen_for);
REQ(n, comp_for);
for_ch = CHILD(ch, 1);
for_ch = CHILD(n, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
return NULL;
expression = ast_for_expr(c, CHILD(ch, 3));
expression = ast_for_expr(c, CHILD(n, 3));
if (!expression)
return NULL;
@ -1258,21 +1254,20 @@ ast_for_genexp(struct compiling *c, const node *n)
(x for x, in ...) has 1 element in t, but still requires a Tuple. */
first = (expr_ty)asdl_seq_GET(t, 0);
if (NCH(for_ch) == 1)
ge = comprehension(first, expression, NULL, c->c_arena);
comp = comprehension(first, expression, NULL, c->c_arena);
else
ge = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!ge)
if (!comp)
return NULL;
if (NCH(ch) == 5) {
if (NCH(n) == 5) {
int j, n_ifs;
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_gen_ifs(c, ch);
n = CHILD(n, 4);
n_ifs = count_comp_ifs(c, n);
if (n_ifs == -1)
return NULL;
@ -1281,32 +1276,94 @@ ast_for_genexp(struct compiling *c, const node *n)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, gen_iter);
ch = CHILD(ch, 0);
REQ(ch, gen_if);
REQ(n, comp_iter);
n = CHILD(n, 0);
REQ(n, comp_if);
expression = ast_for_expr(c, CHILD(ch, 1));
expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
asdl_seq_SET(ifs, j, expression);
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
if (NCH(n) == 3)
n = CHILD(n, 2);
}
/* on exit, must guarantee that ch is a gen_for */
if (TYPE(ch) == gen_iter)
ch = CHILD(ch, 0);
ge->ifs = ifs;
/* on exit, must guarantee that n is a comp_for */
if (TYPE(n) == comp_iter)
n = CHILD(n, 0);
comp->ifs = ifs;
}
asdl_seq_SET(genexps, i, ge);
asdl_seq_SET(comps, i, comp);
}
return comps;
}
static expr_ty
ast_for_itercomp(struct compiling *c, const node *n, int type)
{
expr_ty elt;
asdl_seq *comps;
return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
comps = ast_for_comprehension(c, CHILD(n, 1));
if (!comps)
return NULL;
if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
}
static expr_ty
ast_for_dictcomp(struct compiling *c, const node *n)
{
expr_ty key, value;
asdl_seq *comps;
assert(NCH(n) > 3);
REQ(CHILD(n, 1), COLON);
key = ast_for_expr(c, CHILD(n, 0));
if (!key)
return NULL;
value = ast_for_expr(c, CHILD(n, 2));
if (!value)
return NULL;
comps = ast_for_comprehension(c, CHILD(n, 3));
if (!comps)
return NULL;
return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument));
return ast_for_itercomp(c, n, COMP_GENEXP);
}
static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_itercomp(c, n, COMP_SETCOMP);
}
static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
/* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']'
/* atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']'
| '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
*/
node *ch = CHILD(n, 0);
@ -1365,7 +1422,7 @@ ast_for_atom(struct compiling *c, const node *n)
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
return ast_for_testlist_gexp(c, ch);
return ast_for_testlist_comp(c, ch);
case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
@ -1383,8 +1440,9 @@ ast_for_atom(struct compiling *c, const node *n)
else
return ast_for_listcomp(c, ch);
case LBRACE: {
/* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
* test (',' test)* [','])
/* dictorsetmaker:
* (test ':' test (comp_for | (',' test ':' test)* [','])) |
* (test (comp_for | (',' test)* [',']))
*/
int i, size;
asdl_seq *keys, *values;
@ -1408,6 +1466,11 @@ ast_for_atom(struct compiling *c, const node *n)
asdl_seq_SET(elts, i / 2, expression);
}
return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
} else if (TYPE(CHILD(ch, 1)) == comp_for) {
/* it's a set comprehension */
return ast_for_setcomp(c, ch);
} else if (NCH(ch) > 3 && TYPE(CHILD(ch, 3)) == comp_for) {
return ast_for_dictcomp(c, ch);
} else {
/* it's a dict */
size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
@ -1916,7 +1979,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
/*
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
| '**' test)
argument: [test '='] test [gen_for] # Really [keyword '='] test
argument: [test '='] test [comp_for] # Really [keyword '='] test
*/
int i, nargs, nkeywords, ngens;
@ -1934,7 +1997,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
if (TYPE(ch) == argument) {
if (NCH(ch) == 1)
nargs++;
else if (TYPE(CHILD(ch, 1)) == gen_for)
else if (TYPE(CHILD(ch, 1)) == comp_for)
ngens++;
else
nkeywords++;
@ -1979,7 +2042,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
return NULL;
asdl_seq_SET(args, nargs++, e);
}
else if (TYPE(CHILD(ch, 1)) == gen_for) {
else if (TYPE(CHILD(ch, 1)) == comp_for) {
e = ast_for_genexp(c, ch);
if (!e)
return NULL;
@ -2049,14 +2112,14 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
static expr_ty
ast_for_testlist(struct compiling *c, const node* n)
{
/* testlist_gexp: test (',' test)* [','] */
/* testlist_comp: test (',' test)* [','] */
/* testlist: test (',' test)* [','] */
/* testlist_safe: test (',' test)+ [','] */
/* testlist1: test (',' test)* */
assert(NCH(n) > 0);
if (TYPE(n) == testlist_gexp) {
if (TYPE(n) == testlist_comp) {
if (NCH(n) > 1)
assert(TYPE(CHILD(n, 1)) != gen_for);
assert(TYPE(CHILD(n, 1)) != comp_for);
}
else {
assert(TYPE(n) == testlist ||
@ -2074,12 +2137,12 @@ ast_for_testlist(struct compiling *c, const node* n)
}
static expr_ty
ast_for_testlist_gexp(struct compiling *c, const node* n)
ast_for_testlist_comp(struct compiling *c, const node* n)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] ) */
/* argument: test [ gen_for ] */
assert(TYPE(n) == testlist_gexp || TYPE(n) == argument);
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for)
/* testlist_comp: test ( comp_for | (',' test)* [','] ) */
/* argument: test [ comp_for ] */
assert(TYPE(n) == testlist_comp || TYPE(n) == argument);
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == comp_for)
return ast_for_genexp(c, n);
return ast_for_testlist(c, n);
}

26
Python/ceval.c
View file

@ -1455,6 +1455,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
}
break;
case SET_ADD:
w = POP();
v = stack_pointer[-oparg];
err = PySet_Add(v, w);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
continue;
}
break;
case INPLACE_POWER:
w = POP();
v = TOP();
@ -2223,6 +2234,21 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
if (err == 0) continue;
break;
case MAP_ADD:
w = TOP(); /* key */
u = SECOND(); /* value */
STACKADJ(-2);
v = stack_pointer[-oparg]; /* dict */
assert (PyDict_CheckExact(v));
err = PyDict_SetItem(v, w, u); /* v[w] = u */
Py_DECREF(u);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
continue;
}
break;
case LOAD_ATTR:
w = GETITEM(names, oparg);
v = TOP();

202
Python/compile.c
View file

@ -39,6 +39,10 @@ int Py_OptimizeFlag = 0;
#define DEFAULT_CODE_SIZE 128
#define DEFAULT_LNOTAB_SIZE 16
#define COMP_GENEXP 0
#define COMP_SETCOMP 1
#define COMP_DICTCOMP 2
struct instr {
unsigned i_jabs : 1;
unsigned i_jrel : 1;
@ -674,9 +678,13 @@ opcode_stack_effect(int opcode, int oparg)
case UNARY_INVERT:
return 0;
case SET_ADD:
case LIST_APPEND:
return -1;
case MAP_ADD:
return -2;
case BINARY_POWER:
case BINARY_MULTIPLY:
case BINARY_DIVIDE:
@ -2639,33 +2647,44 @@ compiler_listcomp(struct compiler *c, expr_ty e)
e->v.ListComp.elt);
}
/* Dict and set comprehensions and generator expressions work by creating a
nested function to perform the actual iteration. This means that the
iteration variables don't leak into the current scope.
The defined function is called immediately following its definition, with the
result of that call being the result of the expression.
The LC/SC version returns the populated container, while the GE version is
flagged in symtable.c as a generator, so it returns the generator object
when the function is called.
This code *knows* that the loop cannot contain break, continue, or return,
so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
Possible cleanups:
- iterate over the generator sequence instead of using recursion
*/
static int
compiler_genexp_generator(struct compiler *c,
asdl_seq *generators, int gen_index,
expr_ty elt)
compiler_comprehension_generator(struct compiler *c,
asdl_seq *generators, int gen_index,
expr_ty elt, expr_ty val, int type)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty ge;
basicblock *start, *anchor, *skip, *if_cleanup, *end;
comprehension_ty gen;
basicblock *start, *anchor, *skip, *if_cleanup;
int i, n;
start = compiler_new_block(c);
skip = compiler_new_block(c);
if_cleanup = compiler_new_block(c);
anchor = compiler_new_block(c);
end = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL || end == NULL)
return 0;
ge = (comprehension_ty)asdl_seq_GET(generators, gen_index);
ADDOP_JREL(c, SETUP_LOOP, end);
if (!compiler_push_fblock(c, LOOP, start))
anchor == NULL)
return 0;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1;
@ -2673,77 +2692,164 @@ compiler_genexp_generator(struct compiler *c,
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, ge->iter);
VISIT(c, expr, gen->iter);
ADDOP(c, GET_ITER);
}
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, ge->target);
VISIT(c, expr, gen->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(ge->ifs);
n = asdl_seq_LEN(gen->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i);
expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
VISIT(c, expr, e);
ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_genexp_generator(c, generators, gen_index, elt))
return 0;
if (!compiler_comprehension_generator(c,
generators, gen_index,
elt, val, type))
return 0;
/* only append after the last 'for' generator */
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
/* comprehension specific code */
switch (type) {
case COMP_GENEXP:
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
break;
case COMP_SETCOMP:
VISIT(c, expr, elt);
ADDOP_I(c, SET_ADD, gen_index + 1);
break;
case COMP_DICTCOMP:
/* With 'd[k] = v', v is evaluated before k, so we do
the same. */
VISIT(c, expr, val);
VISIT(c, expr, elt);
ADDOP_I(c, MAP_ADD, gen_index + 1);
break;
default:
return 0;
}
compiler_use_next_block(c, skip);
}
compiler_use_next_block(c, if_cleanup);
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, LOOP, start);
compiler_use_next_block(c, end);
return 1;
}
static int
compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
asdl_seq *generators, expr_ty elt, expr_ty val)
{
PyCodeObject *co = NULL;
expr_ty outermost_iter;
outermost_iter = ((comprehension_ty)
asdl_seq_GET(generators, 0))->iter;
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
goto error;
if (type != COMP_GENEXP) {
int op;
switch (type) {
case COMP_SETCOMP:
op = BUILD_SET;
break;
case COMP_DICTCOMP:
op = BUILD_MAP;
break;
default:
PyErr_Format(PyExc_SystemError,
"unknown comprehension type %d", type);
goto error_in_scope;
}
ADDOP_I(c, op, 0);
}
if (!compiler_comprehension_generator(c, generators, 0, elt,
val, type))
goto error_in_scope;
if (type != COMP_GENEXP) {
ADDOP(c, RETURN_VALUE);
}
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
goto error;
if (!compiler_make_closure(c, co, 0))
goto error;
Py_DECREF(co);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
error_in_scope:
compiler_exit_scope(c);
error:
Py_XDECREF(co);
return 0;
}
static int
compiler_genexp(struct compiler *c, expr_ty e)
{
static identifier name;
PyCodeObject *co;
expr_ty outermost_iter = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators,
0)))->iter;
if (!name) {
name = PyString_FromString("<genexpr>");
if (!name)
return 0;
}
assert(e->kind == GeneratorExp_kind);
return compiler_comprehension(c, e, COMP_GENEXP, name,
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt, NULL);
}
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
return 0;
compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0,
e->v.GeneratorExp.elt);
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
return 0;
static int
compiler_setcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<setcomp>");
if (!name)
return 0;
}
assert(e->kind == SetComp_kind);
return compiler_comprehension(c, e, COMP_SETCOMP, name,
e->v.SetComp.generators,
e->v.SetComp.elt, NULL);
}
compiler_make_closure(c, co, 0);
Py_DECREF(co);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
static int
compiler_dictcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<dictcomp>");
if (!name)
return 0;
}
assert(e->kind == DictComp_kind);
return compiler_comprehension(c, e, COMP_DICTCOMP, name,
e->v.DictComp.generators,
e->v.DictComp.key, e->v.DictComp.value);
}
static int
@ -2902,6 +3008,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
break;
case ListComp_kind:
return compiler_listcomp(c, e);
case SetComp_kind:
return compiler_setcomp(c, e);
case DictComp_kind:
return compiler_dictcomp(c, e);
case GeneratorExp_kind:
return compiler_genexp(c, e);
case Yield_kind:

57
Python/graminit.c
View file

@ -1550,42 +1550,57 @@ static state states_72[5] = {
static arc arcs_73_0[1] = {
{28, 1},
};
static arc arcs_73_1[3] = {
static arc arcs_73_1[4] = {
{23, 2},
{29, 3},
{157, 3},
{29, 4},
{0, 1},
};
static arc arcs_73_2[1] = {
{28, 4},
};
static arc arcs_73_3[2] = {
{28, 5},
};
static arc arcs_73_3[1] = {
{0, 3},
};
static arc arcs_73_4[2] = {
{29, 6},
{28, 6},
{0, 4},
};
static arc arcs_73_5[2] = {
{29, 3},
static arc arcs_73_5[3] = {
{157, 3},
{29, 7},
{0, 5},
};
static arc arcs_73_6[2] = {
{28, 7},
{29, 4},
{0, 6},
};
static arc arcs_73_7[1] = {
{23, 2},
static arc arcs_73_7[2] = {
{28, 8},
{0, 7},
};
static state states_73[8] = {
static arc arcs_73_8[1] = {
{23, 9},
};
static arc arcs_73_9[1] = {
{28, 10},
};
static arc arcs_73_10[2] = {
{29, 7},
{0, 10},
};
static state states_73[11] = {
{1, arcs_73_0},
{3, arcs_73_1},
{4, arcs_73_1},
{1, arcs_73_2},
{2, arcs_73_3},
{1, arcs_73_3},
{2, arcs_73_4},
{2, arcs_73_5},
{3, arcs_73_5},
{2, arcs_73_6},
{1, arcs_73_7},
{2, arcs_73_7},
{1, arcs_73_8},
{1, arcs_73_9},
{2, arcs_73_10},
};
static arc arcs_74_0[1] = {
{162, 1},
@ -1964,7 +1979,7 @@ static dfa dfas[86] = {
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"},
{319, "listmaker", 0, 5, states_63,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{320, "testlist_gexp", 0, 5, states_64,
{320, "testlist_comp", 0, 5, states_64,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{321, "lambdef", 0, 5, states_65,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
@ -1982,7 +1997,7 @@ static dfa dfas[86] = {
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{328, "dictmaker", 0, 5, states_72,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{329, "dictorsetmaker", 0, 8, states_73,
{329, "dictorsetmaker", 0, 11, states_73,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{330, "classdef", 0, 8, states_74,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000"},
@ -1996,11 +2011,11 @@ static dfa dfas[86] = {
"\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
{335, "list_if", 0, 4, states_79,
"\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"},
{336, "gen_iter", 0, 2, states_80,
{336, "comp_iter", 0, 2, states_80,
"\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"},
{337, "gen_for", 0, 6, states_81,
{337, "comp_for", 0, 6, states_81,
"\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
{338, "gen_if", 0, 4, states_82,
{338, "comp_if", 0, 4, states_82,
"\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"},
{339, "testlist1", 0, 2, states_83,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},

3
Python/import.c
View file

@ -76,9 +76,10 @@ typedef unsigned short mode_t;
introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
Python 2.7a0 62191 (introduce SETUP_WITH)
Python 2.7a0 62201 (introduce BUILD_SET)
Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
.
*/
#define MAGIC (62201 | ((long)'\r'<<16) | ((long)'\n'<<24))
#define MAGIC (62211 | ((long)'\r'<<16) | ((long)'\n'<<24))
/* Magic word as global; note that _PyImport_Init() can change the
value of this global to accommodate for alterations of how the

84
Python/symtable.c
View file

@ -166,6 +166,8 @@ static int symtable_exit_block(struct symtable *st, void *ast);
static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
static int symtable_visit_expr(struct symtable *st, expr_ty s);
static int symtable_visit_genexp(struct symtable *st, expr_ty s);
static int symtable_visit_setcomp(struct symtable *st, expr_ty e);
static int symtable_visit_dictcomp(struct symtable *st, expr_ty e);
static int symtable_visit_arguments(struct symtable *st, arguments_ty);
static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
static int symtable_visit_alias(struct symtable *st, alias_ty);
@ -177,7 +179,8 @@ static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args);
static int symtable_implicit_arg(struct symtable *st, int pos);
static identifier top = NULL, lambda = NULL, genexpr = NULL;
static identifier top = NULL, lambda = NULL, genexpr = NULL, setcomp = NULL,
dictcomp = NULL;
#define GET_IDENTIFIER(VAR) \
((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR)))
@ -1222,6 +1225,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
if (!symtable_visit_genexp(st, e))
return 0;
break;
case SetComp_kind:
if (!symtable_visit_setcomp(st, e))
return 0;
break;
case DictComp_kind:
if (!symtable_visit_dictcomp(st, e))
return 0;
break;
case Yield_kind:
if (e->v.Yield.value)
VISIT(st, expr, e->v.Yield.value);
@ -1462,28 +1473,81 @@ symtable_visit_slice(struct symtable *st, slice_ty s)
return 1;
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
static int
symtable_new_tmpname(struct symtable *st)
{
char tmpname[256];
identifier tmp;
PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]",
++st->st_cur->ste_tmpname);
tmp = PyString_InternFromString(tmpname);
if (!tmp)
return 0;
if (!symtable_add_def(st, tmp, DEF_LOCAL))
return 0;
Py_DECREF(tmp);
return 1;
}
static int
symtable_handle_comprehension(struct symtable *st, expr_ty e,
identifier scope_name, asdl_seq *generators,
expr_ty elt, expr_ty value)
{
int is_generator = (e->kind == GeneratorExp_kind);
int needs_tmp = !is_generator;
comprehension_ty outermost = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators, 0)));
asdl_seq_GET(generators, 0));
/* Outermost iterator is evaluated in current scope */
VISIT(st, expr, outermost->iter);
/* Create generator scope for the rest */
if (!GET_IDENTIFIER(genexpr) ||
!symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, e->lineno)) {
/* Create comprehension scope for the rest */
if (!scope_name ||
!symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
return 0;
}
st->st_cur->ste_generator = 1;
st->st_cur->ste_generator = is_generator;
/* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) {
symtable_exit_block(st, (void *)e);
return 0;
}
/* Allocate temporary name if needed */
if (needs_tmp && !symtable_new_tmpname(st)) {
symtable_exit_block(st, (void *)e);
return 0;
}
VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e);
VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e);
VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension,
e->v.GeneratorExp.generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e);
generators, 1, (void*)e);
if (value)
VISIT_IN_BLOCK(st, expr, value, (void*)e);
VISIT_IN_BLOCK(st, expr, elt, (void*)e);
return symtable_exit_block(st, (void *)e);
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt, NULL);
}
static int
symtable_visit_setcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
e->v.SetComp.generators,
e->v.SetComp.elt, NULL);
}
static int
symtable_visit_dictcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(dictcomp),
e->v.DictComp.generators,
e->v.DictComp.key,
e->v.DictComp.value);
}