mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 05:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Hide list comp variables and support set comprehensions

Browse source
This commit is contained in:
Nick Coghlan 2007-04-15 12:05:43 +00:00
parent 6ef6306dd6
commit 650f0d06d3
29 changed files with 2006 additions and 1323 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Doc/lib/libdis.tex
View file

@ -363,6 +363,10 @@ is the address to jump to (which should be a \code{FOR_ITER}
instruction). instruction).
\end{opcodedesc} \end{opcodedesc}
\begin{opcodedesc}{SET_ADD}{}
Calls \code{set.add(TOS1, TOS)}. Used to implement set comprehensions.
\end{opcodedesc}
\begin{opcodedesc}{LIST_APPEND}{} \begin{opcodedesc}{LIST_APPEND}{}
Calls \code{list.append(TOS1, TOS)}. Used to implement list comprehensions. Calls \code{list.append(TOS1, TOS)}. Used to implement list comprehensions.
\end{opcodedesc} \end{opcodedesc}

37
Grammar/Grammar
View file

@ -82,16 +82,10 @@ with_var: 'as' expr
except_clause: 'except' [test ['as' NAME]] except_clause: 'except' [test ['as' NAME]]
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
# Backward compatibility cruft to support:
# [ x for x in lambda: True, lambda: False if x() ]
# even while also allowing:
# lambda x: 5 if x else 2
# (But not a mix of the two)
testlist_safe: old_test [(',' old_test)+ [',']]
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
test: or_test ['if' or_test 'else' test] | lambdef test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
or_test: and_test ('or' and_test)* or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)* and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison not_test: 'not' not_test | comparison
@ -105,33 +99,28 @@ arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)* term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor] power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' | atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [listmaker] ']' | '[' [testlist_comp] ']' |
'{' [dictsetmaker] '}' | '{' [dictorsetmaker] '}' |
NAME | NUMBER | STRING+ | '...') NAME | NUMBER | STRING+ | '...')
listmaker: test ( list_for | (',' test)* [','] ) testlist_comp: test ( comp_for | (',' test)* [','] )
testlist_gexp: test ( gen_for | (',' test)* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [','] subscriptlist: subscript (',' subscript)* [',']
subscript: test | [test] ':' [test] [sliceop] subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test] sliceop: ':' [test]
exprlist: expr (',' expr)* [','] exprlist: expr (',' expr)* [',']
testlist: test (',' test)* [','] testlist: test (',' test)* [',']
dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) dictorsetmaker: ( (test ':' test (',' test ':' test)* [',']) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
argument: test [gen_for] | test '=' test # Really [keyword '='] test argument: test [comp_for] | test '=' test # Really [keyword '='] test
list_iter: list_for | list_if comp_iter: comp_for | comp_if
list_for: 'for' exprlist 'in' testlist_safe [list_iter] comp_for: 'for' exprlist 'in' or_test [comp_iter]
list_if: 'if' old_test [list_iter] comp_if: 'if' test_nocond [comp_iter]
gen_iter: gen_for | gen_if
gen_for: 'for' exprlist 'in' or_test [gen_iter]
gen_if: 'if' old_test [gen_iter]
testlist1: test (',' test)* testlist1: test (',' test)*

16
Include/Python-ast.h
View file

@ -183,10 +183,10 @@ struct _stmt {
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
GeneratorExp_kind=9, Yield_kind=10, Compare_kind=11, SetComp_kind=9, GeneratorExp_kind=10, Yield_kind=11,
Call_kind=12, Num_kind=13, Str_kind=14, Bytes_kind=15, Compare_kind=12, Call_kind=13, Num_kind=14, Str_kind=15,
Ellipsis_kind=16, Attribute_kind=17, Subscript_kind=18, Bytes_kind=16, Ellipsis_kind=17, Attribute_kind=18,
Name_kind=19, List_kind=20, Tuple_kind=21}; Subscript_kind=19, Name_kind=20, List_kind=21, Tuple_kind=22};
struct _expr { struct _expr {
enum _expr_kind kind; enum _expr_kind kind;
union { union {
@ -231,6 +231,11 @@ struct _expr {
asdl_seq *generators; asdl_seq *generators;
} ListComp; } ListComp;
struct {
expr_ty elt;
asdl_seq *generators;
} SetComp;
struct { struct {
expr_ty elt; expr_ty elt;
asdl_seq *generators; asdl_seq *generators;
@ -465,6 +470,9 @@ expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4) #define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena); col_offset, PyArena *arena);
#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena); col_offset, PyArena *arena);

47
Include/graminit.h
View file

@ -46,10 +46,10 @@
#define with_var 301 #define with_var 301
#define except_clause 302 #define except_clause 302
#define suite 303 #define suite 303
#define testlist_safe 304 #define test 304
#define old_test 305 #define test_nocond 305
#define old_lambdef 306 #define lambdef 306
#define test 307 #define lambdef_nocond 307
#define or_test 308 #define or_test 308
#define and_test 309 #define and_test 309
#define not_test 310 #define not_test 310
@ -64,25 +64,20 @@
#define factor 319 #define factor 319
#define power 320 #define power 320
#define atom 321 #define atom 321
#define listmaker 322 #define testlist_comp 322
#define testlist_gexp 323 #define trailer 323
#define lambdef 324 #define subscriptlist 324
#define trailer 325 #define subscript 325
#define subscriptlist 326 #define sliceop 326
#define subscript 327 #define exprlist 327
#define sliceop 328 #define testlist 328
#define exprlist 329 #define dictorsetmaker 329
#define testlist 330 #define classdef 330
#define dictsetmaker 331 #define arglist 331
#define classdef 332 #define argument 332
#define arglist 333 #define comp_iter 333
#define argument 334 #define comp_for 334
#define list_iter 335 #define comp_if 335
#define list_for 336 #define testlist1 336
#define list_if 337 #define encoding_decl 337
#define gen_iter 338 #define yield_expr 338
#define gen_for 339
#define gen_if 340
#define testlist1 341
#define encoding_decl 342
#define yield_expr 343

1
Include/opcode.h
View file

@ -21,6 +21,7 @@ extern "C" {
#define UNARY_INVERT 15 #define UNARY_INVERT 15
#define SET_ADD 17
#define LIST_APPEND 18 #define LIST_APPEND 18
#define BINARY_POWER 19 #define BINARY_POWER 19

32
Include/symtable.h
View file

@ -5,31 +5,35 @@
extern "C" { extern "C" {
#endif #endif
/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal
* names.
*/
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
_Py_block_ty; _Py_block_ty;
struct _symtable_entry; struct _symtable_entry;
struct symtable { struct symtable {
const char *st_filename; /* name of file being compiled */ const char *st_filename; /* name of file being compiled */
struct _symtable_entry *st_cur; /* current symbol table entry */ struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* module entry */ struct _symtable_entry *st_top; /* symbol table entry for module */
PyObject *st_symbols; /* dictionary of symbol table entries */ PyObject *st_blocks; /* dict: map AST node addresses
PyObject *st_stack; /* stack of namespace info */ * to symbol table entries */
PyObject *st_global; /* borrowed ref to MODULE in st_symbols */ PyObject *st_stack; /* list: stack of namespace info */
int st_nblocks; /* number of blocks */ PyObject *st_global; /* borrowed ref to st_top->st_symbols */
PyObject *st_private; /* name of current class or NULL */ int st_nblocks; /* number of blocks used */
int st_tmpname; /* temporary name counter */ PyObject *st_private; /* name of current class or NULL */
PyFutureFeatures *st_future; /* module's future features */ PyFutureFeatures *st_future; /* module's future features */
}; };
typedef struct _symtable_entry { typedef struct _symtable_entry {
PyObject_HEAD PyObject_HEAD
PyObject *ste_id; /* int: key in st_symbols */ PyObject *ste_id; /* int: key in ste_table->st_blocks */
PyObject *ste_symbols; /* dict: name to flags */ PyObject *ste_symbols; /* dict: variable names to flags */
PyObject *ste_name; /* string: name of block */ PyObject *ste_name; /* string: name of current block */
PyObject *ste_varnames; /* list of variable names */ PyObject *ste_varnames; /* list of variable names */
PyObject *ste_children; /* list of child ids */ PyObject *ste_children; /* list of child blocks */
_Py_block_ty ste_type; /* module, class, or function */ _Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */ int ste_unoptimized; /* false if namespace is optimized */
unsigned ste_nested : 1; /* true if block is nested */ unsigned ste_nested : 1; /* true if block is nested */
@ -80,7 +84,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
table. GLOBAL is returned from PyST_GetScope() for either of them. table. GLOBAL is returned from PyST_GetScope() for either of them.
It is stored in ste_symbols at bits 12-15. It is stored in ste_symbols at bits 12-15.
*/ */
#define SCOPE_OFF 11 #define SCOPE_OFFSET 11
#define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL) #define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL)
#define LOCAL 1 #define LOCAL 1

4
Lib/compiler/transformer.py
View file

@ -559,7 +559,7 @@ class Transformer:
testlist1 = testlist testlist1 = testlist
exprlist = testlist exprlist = testlist
def testlist_gexp(self, nodelist): def testlist_comp(self, nodelist):
if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for:
test = self.com_node(nodelist[0]) test = self.com_node(nodelist[0])
return self.com_generator_expression(test, nodelist[1]) return self.com_generator_expression(test, nodelist[1])
@ -1027,7 +1027,7 @@ class Transformer:
# loop to avoid trivial recursion # loop to avoid trivial recursion
while 1: while 1:
t = node[0] t = node[0]
if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp): if t in (symbol.exprlist, symbol.testlist, symbol.testlist_comp):
if len(node) > 2: if len(node) > 2:
return self.com_assign_tuple(node, assigning) return self.com_assign_tuple(node, assigning)
node = node[1] node = node[1]

2
Lib/decimal.py
View file

@ -2282,10 +2282,8 @@ class Context(object):
_ignored_flags = [] _ignored_flags = []
if not isinstance(flags, dict): if not isinstance(flags, dict):
flags = dict([(s,s in flags) for s in _signals]) flags = dict([(s,s in flags) for s in _signals])
del s
if traps is not None and not isinstance(traps, dict): if traps is not None and not isinstance(traps, dict):
traps = dict([(s,s in traps) for s in _signals]) traps = dict([(s,s in traps) for s in _signals])
del s
for name, val in locals().items(): for name, val in locals().items():
if val is None: if val is None:
setattr(self, name, _copy.copy(getattr(DefaultContext, name))) setattr(self, name, _copy.copy(getattr(DefaultContext, name)))

1
Lib/opcode.py
View file

@ -57,6 +57,7 @@ def_op('UNARY_NOT', 12)
def_op('UNARY_INVERT', 15) def_op('UNARY_INVERT', 15)
def_op('SET_ADD', 17)
def_op('LIST_APPEND', 18) def_op('LIST_APPEND', 18)
def_op('BINARY_POWER', 19) def_op('BINARY_POWER', 19)
def_op('BINARY_MULTIPLY', 20) def_op('BINARY_MULTIPLY', 20)

1
Lib/pickle.py
View file

@ -163,7 +163,6 @@ _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
del x
# Pickling machinery # Pickling machinery

4
Lib/test/test_datetime.py
View file

@ -20,9 +20,9 @@ from datetime import time
from datetime import date, datetime from datetime import date, datetime
pickle_choices = [(pickler, unpickler, proto) pickle_choices = [(pickler, unpickler, proto)
for pickler in pickle, cPickle for pickler in (pickle, cPickle)
if pickler is not None if pickler is not None
for unpickler in pickle, cPickle for unpickler in (pickle, cPickle)
if unpickler is not None if unpickler is not None
for proto in range(3)] for proto in range(3)]
if cPickle is None: if cPickle is None:

8
Lib/test/test_dis.py
View file

@ -129,8 +129,12 @@ class DisTests(unittest.TestCase):
def test_bug_1333982(self): def test_bug_1333982(self):
# This one is checking bytecodes generated for an `assert` statement, # This one is checking bytecodes generated for an `assert` statement,
# so fails if the tests are run with -O. Skip this test then. # so fails if the tests are run with -O. Skip this test then.
if __debug__: pass # Test has been disabled due to change in the way
self.do_disassembly_test(bug1333982, dis_bug1333982) # list comps are handled. The byte code now includes
# a memory address and a file location, so they change from
# run to run.
# if __debug__:
# self.do_disassembly_test(bug1333982, dis_bug1333982)
def test_big_linenos(self): def test_big_linenos(self):
def func(count): def func(count):

3
Lib/test/test_grammar.py
View file

@ -843,7 +843,8 @@ class GrammarTests(unittest.TestCase):
print(x) print(x)
return ret return ret
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) # the next line is not allowed anymore
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)

444
Lib/test/test_listcomps.py Normal file
View file

@ -0,0 +1,444 @@
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)

453
Lib/test/test_setcomps.py Normal file
View file

@ -0,0 +1,453 @@
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)

4
Lib/test/test_syntax.py
View file

@ -5,7 +5,7 @@ Here's an example of the sort of thing that is tested.
>>> def f(x): >>> def f(x):
... global x ... global x
Traceback (most recent call last): Traceback (most recent call last):
SyntaxError: name 'x' is local and global SyntaxError: name 'x' is parameter and global
The tests are all raise SyntaxErrors. They were created by checking The tests are all raise SyntaxErrors. They were created by checking
each C call that raises SyntaxError. There are several modules that each C call that raises SyntaxError. There are several modules that
@ -373,7 +373,7 @@ Misuse of the nonlocal statement can lead to a few unique syntax errors.
... nonlocal x ... nonlocal x
Traceback (most recent call last): Traceback (most recent call last):
... ...
SyntaxError: name 'x' is local and nonlocal SyntaxError: name 'x' is parameter and nonlocal
>>> def f(): >>> def f():
... global x ... global x

1
Lib/test/test_univnewlines.py
View file

@ -28,7 +28,6 @@ DATA_CRLF = "\r\n".join(DATA_TEMPLATE) + "\r\n"
# before end-of-file. # before end-of-file.
DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r" DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r"
DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE] DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE]
del x
class TestGenericUnivNewlines(unittest.TestCase): class TestGenericUnivNewlines(unittest.TestCase):
# use a class variable DATA to define the data to write to the file # use a class variable DATA to define the data to write to the file

1
Lib/tokenize.py
View file

@ -32,7 +32,6 @@ from token import *
import token import token
__all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize", __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",
"generate_tokens", "NL", "untokenize"] "generate_tokens", "NL", "untokenize"]
del x
del token del token
COMMENT = N_TOKENS COMMENT = N_TOKENS

3
Misc/NEWS
View file

@ -28,6 +28,9 @@ TO DO
Core and Builtins Core and Builtins
----------------- -----------------
- Patch #1660500: hide iteration variable in list comps, add set comps
and use common code to handle compilation of iterative expressions
- By default, != returns the opposite of ==, unless the latter returns - By default, != returns the opposite of ==, unless the latter returns
NotImplemented. NotImplemented.

205
Modules/parsermodule.c
View file

@ -858,11 +858,10 @@ VALIDATER(vfpdef); VALIDATER(vfplist);
VALIDATER(stmt); VALIDATER(simple_stmt); VALIDATER(stmt); VALIDATER(simple_stmt);
VALIDATER(expr_stmt); VALIDATER(power); VALIDATER(expr_stmt); VALIDATER(power);
VALIDATER(del_stmt); VALIDATER(del_stmt);
VALIDATER(return_stmt); VALIDATER(list_iter); VALIDATER(return_stmt); VALIDATER(raise_stmt);
VALIDATER(raise_stmt); VALIDATER(import_stmt); VALIDATER(import_stmt); VALIDATER(import_stmt);
VALIDATER(import_name); VALIDATER(import_from); VALIDATER(import_name); VALIDATER(yield_stmt);
VALIDATER(global_stmt); VALIDATER(list_if); VALIDATER(global_stmt); VALIDATER(assert_stmt);
VALIDATER(assert_stmt); VALIDATER(list_for);
VALIDATER(compound_stmt); VALIDATER(vname); VALIDATER(compound_stmt); VALIDATER(vname);
VALIDATER(while); VALIDATER(for); VALIDATER(while); VALIDATER(for);
VALIDATER(try); VALIDATER(except_clause); VALIDATER(try); VALIDATER(except_clause);
@ -875,14 +874,13 @@ VALIDATER(term); VALIDATER(factor);
VALIDATER(atom); VALIDATER(lambdef); VALIDATER(atom); VALIDATER(lambdef);
VALIDATER(trailer); VALIDATER(subscript); VALIDATER(trailer); VALIDATER(subscript);
VALIDATER(subscriptlist); VALIDATER(sliceop); VALIDATER(subscriptlist); VALIDATER(sliceop);
VALIDATER(exprlist); VALIDATER(dictsetmaker); VALIDATER(exprlist); VALIDATER(dictorsetmaker);
VALIDATER(arglist); VALIDATER(argument); VALIDATER(arglist); VALIDATER(argument);
VALIDATER(listmaker); VALIDATER(yield_stmt); VALIDATER(testlist1); VALIDATER(comp_for);
VALIDATER(testlist1); VALIDATER(gen_for); VALIDATER(comp_iter); VALIDATER(comp_if);
VALIDATER(gen_iter); VALIDATER(gen_if); VALIDATER(testlist_comp); VALIDATER(yield_expr);
VALIDATER(testlist_gexp); VALIDATER(yield_expr); VALIDATER(yield_or_testlist); VALIDATER(or_test);
VALIDATER(yield_or_testlist); VALIDATER(or_test); VALIDATER(test_nocond); VALIDATER(lambdef_nocond);
VALIDATER(old_test); VALIDATER(old_lambdef);
#undef VALIDATER #undef VALIDATER
@ -1112,14 +1110,6 @@ validate_testlist1(node *tree)
} }
static int
validate_testlist_safe(node *tree)
{
return (validate_repeating_list(tree, testlist_safe,
validate_old_test, "testlist_safe"));
}
/* validate either vname or tname. /* validate either vname or tname.
* vname: NAME * vname: NAME
* tname: NAME [':' test] * tname: NAME [':' test]
@ -1330,70 +1320,33 @@ validate_varargslist(node *tree)
} }
/* list_iter: list_for | list_if /* comp_iter: comp_for | comp_if
*/ */
static int static int
validate_list_iter(node *tree) validate_comp_iter(node *tree)
{ {
int res = (validate_ntype(tree, list_iter) int res = (validate_ntype(tree, comp_iter)
&& validate_numnodes(tree, 1, "list_iter")); && validate_numnodes(tree, 1, "comp_iter"));
if (res && TYPE(CHILD(tree, 0)) == list_for) if (res && TYPE(CHILD(tree, 0)) == comp_for)
res = validate_list_for(CHILD(tree, 0)); res = validate_comp_for(CHILD(tree, 0));
else else
res = validate_list_if(CHILD(tree, 0)); res = validate_comp_if(CHILD(tree, 0));
return res; return res;
} }
/* gen_iter: gen_for | gen_if /* comp_for: 'for' exprlist 'in' test [comp_iter]
*/ */
static int static int
validate_gen_iter(node *tree) validate_comp_for(node *tree)
{
int res = (validate_ntype(tree, gen_iter)
&& validate_numnodes(tree, 1, "gen_iter"));
if (res && TYPE(CHILD(tree, 0)) == gen_for)
res = validate_gen_for(CHILD(tree, 0));
else
res = validate_gen_if(CHILD(tree, 0));
return res;
}
/* list_for: 'for' exprlist 'in' testlist [list_iter]
*/
static int
validate_list_for(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res; int res;
if (nch == 5) if (nch == 5)
res = validate_list_iter(CHILD(tree, 4)); res = validate_comp_iter(CHILD(tree, 4));
else else
res = validate_numnodes(tree, 4, "list_for"); res = validate_numnodes(tree, 4, "comp_for");
if (res)
res = (validate_name(CHILD(tree, 0), "for")
&& validate_exprlist(CHILD(tree, 1))
&& validate_name(CHILD(tree, 2), "in")
&& validate_testlist_safe(CHILD(tree, 3)));
return res;
}
/* gen_for: 'for' exprlist 'in' test [gen_iter]
*/
static int
validate_gen_for(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 5)
res = validate_gen_iter(CHILD(tree, 4));
else
res = validate_numnodes(tree, 4, "gen_for");
if (res) if (res)
res = (validate_name(CHILD(tree, 0), "for") res = (validate_name(CHILD(tree, 0), "for")
@ -1404,45 +1357,26 @@ validate_gen_for(node *tree)
return res; return res;
} }
/* list_if: 'if' old_test [list_iter] /* comp_if: 'if' test_nocond [comp_iter]
*/ */
static int static int
validate_list_if(node *tree) validate_comp_if(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res; int res;
if (nch == 3) if (nch == 3)
res = validate_list_iter(CHILD(tree, 2)); res = validate_comp_iter(CHILD(tree, 2));
else else
res = validate_numnodes(tree, 2, "list_if"); res = validate_numnodes(tree, 2, "comp_if");
if (res) if (res)
res = (validate_name(CHILD(tree, 0), "if") res = (validate_name(CHILD(tree, 0), "if")
&& validate_old_test(CHILD(tree, 1))); && validate_test_nocond(CHILD(tree, 1)));
return res; return res;
} }
/* gen_if: 'if' old_test [gen_iter]
*/
static int
validate_gen_if(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 3)
res = validate_gen_iter(CHILD(tree, 2));
else
res = validate_numnodes(tree, 2, "gen_if");
if (res)
res = (validate_name(CHILD(tree, 0), "if")
&& validate_old_test(CHILD(tree, 1)));
return res;
}
/* validate_vfpdef() /* validate_vfpdef()
* *
@ -2089,13 +2023,13 @@ validate_test(node *tree)
} }
static int static int
validate_old_test(node *tree) validate_test_nocond(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res = validate_ntype(tree, old_test) && (nch == 1); int res = validate_ntype(tree, test_nocond) && (nch == 1);
if (res && (TYPE(CHILD(tree, 0)) == old_lambdef)) if (res && (TYPE(CHILD(tree, 0)) == lambdef_nocond))
res = (validate_old_lambdef(CHILD(tree, 0))); res = (validate_lambdef_nocond(CHILD(tree, 0)));
else if (res) { else if (res) {
res = (validate_or_test(CHILD(tree, 0))); res = (validate_or_test(CHILD(tree, 0)));
} }
@ -2393,14 +2327,14 @@ validate_atom(node *tree)
if (TYPE(CHILD(tree, 1))==yield_expr) if (TYPE(CHILD(tree, 1))==yield_expr)
res = validate_yield_expr(CHILD(tree, 1)); res = validate_yield_expr(CHILD(tree, 1));
else else
res = validate_testlist_gexp(CHILD(tree, 1)); res = validate_testlist_comp(CHILD(tree, 1));
} }
break; break;
case LSQB: case LSQB:
if (nch == 2) if (nch == 2)
res = validate_ntype(CHILD(tree, 1), RSQB); res = validate_ntype(CHILD(tree, 1), RSQB);
else if (nch == 3) else if (nch == 3)
res = (validate_listmaker(CHILD(tree, 1)) res = (validate_testlist_comp(CHILD(tree, 1))
&& validate_ntype(CHILD(tree, 2), RSQB)); && validate_ntype(CHILD(tree, 2), RSQB));
else { else {
res = 0; res = 0;
@ -2412,7 +2346,7 @@ validate_atom(node *tree)
&& validate_ntype(CHILD(tree, nch - 1), RBRACE)); && validate_ntype(CHILD(tree, nch - 1), RBRACE));
if (res && (nch == 3)) if (res && (nch == 3))
res = validate_dictsetmaker(CHILD(tree, 1)); res = validate_dictorsetmaker(CHILD(tree, 1));
break; break;
case NAME: case NAME:
case NUMBER: case NUMBER:
@ -2436,25 +2370,26 @@ validate_atom(node *tree)
} }
/* listmaker: /* testlist_comp:
* test ( list_for | (',' test)* [','] ) * test ( comp_for | (',' test)* [','] )
*/ */
static int static int
validate_listmaker(node *tree) validate_testlist_comp(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int ok = nch; int ok = nch;
if (nch == 0) if (nch == 0)
err_string("missing child nodes of listmaker"); err_string("missing child nodes of testlist_comp");
else else {
ok = validate_test(CHILD(tree, 0)); ok = validate_test(CHILD(tree, 0));
}
/* /*
* list_for | (',' test)* [','] * comp_for | (',' test)* [',']
*/ */
if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for) if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for)
ok = validate_list_for(CHILD(tree, 1)); ok = validate_comp_for(CHILD(tree, 1));
else { else {
/* (',' test)* [','] */ /* (',' test)* [','] */
int i = 1; int i = 1;
@ -2467,45 +2402,7 @@ validate_listmaker(node *tree)
ok = validate_comma(CHILD(tree, i)); ok = validate_comma(CHILD(tree, i));
else if (i != nch) { else if (i != nch) {
ok = 0; ok = 0;
err_string("illegal trailing nodes for listmaker"); err_string("illegal trailing nodes for testlist_comp");
}
}
return ok;
}
/* testlist_gexp:
* test ( gen_for | (',' test)* [','] )
*/
static int
validate_testlist_gexp(node *tree)
{
int nch = NCH(tree);
int ok = nch;
if (nch == 0)
err_string("missing child nodes of testlist_gexp");
else {
ok = validate_test(CHILD(tree, 0));
}
/*
* gen_for | (',' test)* [',']
*/
if (nch == 2 && TYPE(CHILD(tree, 1)) == gen_for)
ok = validate_gen_for(CHILD(tree, 1));
else {
/* (',' test)* [','] */
int i = 1;
while (ok && nch - i >= 2) {
ok = (validate_comma(CHILD(tree, i))
&& validate_test(CHILD(tree, i+1)));
i += 2;
}
if (ok && i == nch-1)
ok = validate_comma(CHILD(tree, i));
else if (i != nch) {
ok = 0;
err_string("illegal trailing nodes for testlist_gexp");
} }
} }
return ok; return ok;
@ -2596,10 +2493,10 @@ validate_lambdef(node *tree)
static int static int
validate_old_lambdef(node *tree) validate_lambdef_nocond(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res = (validate_ntype(tree, old_lambdef) int res = (validate_ntype(tree, lambdef_nocond)
&& ((nch == 3) || (nch == 4)) && ((nch == 3) || (nch == 4))
&& validate_name(CHILD(tree, 0), "lambda") && validate_name(CHILD(tree, 0), "lambda")
&& validate_colon(CHILD(tree, nch - 2)) && validate_colon(CHILD(tree, nch - 2))
@ -2608,7 +2505,7 @@ validate_old_lambdef(node *tree)
if (res && (nch == 4)) if (res && (nch == 4))
res = validate_varargslist(CHILD(tree, 1)); res = validate_varargslist(CHILD(tree, 1));
else if (!res && !PyErr_Occurred()) else if (!res && !PyErr_Occurred())
(void) validate_numnodes(tree, 3, "old_lambdef"); (void) validate_numnodes(tree, 3, "lambdef_nocond");
return (res); return (res);
} }
@ -2633,7 +2530,7 @@ validate_arglist(node *tree)
for (i=0; i<nch; i++) { for (i=0; i<nch; i++) {
if (TYPE(CHILD(tree, i)) == argument) { if (TYPE(CHILD(tree, i)) == argument) {
node *ch = CHILD(tree, i); node *ch = CHILD(tree, i);
if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == gen_for) { if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == comp_for) {
err_string("need '(', ')' for generator expression"); err_string("need '(', ')' for generator expression");
return 0; return 0;
} }
@ -2700,7 +2597,7 @@ validate_arglist(node *tree)
/* argument: /* argument:
* *
* [test '='] test [gen_for] * [test '='] test [comp_for]
*/ */
static int static int
validate_argument(node *tree) validate_argument(node *tree)
@ -2711,7 +2608,7 @@ validate_argument(node *tree)
&& validate_test(CHILD(tree, 0))); && validate_test(CHILD(tree, 0)));
if (res && (nch == 2)) if (res && (nch == 2))
res = validate_gen_for(CHILD(tree, 1)); res = validate_comp_for(CHILD(tree, 1));
else if (res && (nch == 3)) else if (res && (nch == 3))
res = (validate_equal(CHILD(tree, 1)) res = (validate_equal(CHILD(tree, 1))
&& validate_test(CHILD(tree, 2))); && validate_test(CHILD(tree, 2)));
@ -2853,10 +2750,10 @@ validate_exprlist(node *tree)
static int static int
validate_dictsetmaker(node *tree) validate_dictorsetmaker(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res = (validate_ntype(tree, dictsetmaker) int res = (validate_ntype(tree, dictorsetmaker)
&& (nch >= 3) && (nch >= 3)
&& validate_test(CHILD(tree, 0)) && validate_test(CHILD(tree, 0))
&& validate_colon(CHILD(tree, 1)) && validate_colon(CHILD(tree, 1))

2
Modules/symtablemodule.c
View file

@ -33,7 +33,7 @@ symtable_symtable(PyObject *self, PyObject *args)
st = Py_SymtableString(str, filename, start); st = Py_SymtableString(str, filename, start);
if (st == NULL) if (st == NULL)
return NULL; return NULL;
t = st->st_symbols; t = st->st_blocks;
Py_INCREF(t); Py_INCREF(t);
PyMem_Free((void *)st->st_future); PyMem_Free((void *)st->st_future);
PySymtable_Free(st); PySymtable_Free(st);

1
Parser/Python.asdl
View file

@ -56,6 +56,7 @@ module Python version "$Revision$"
| Dict(expr* keys, expr* values) | Dict(expr* keys, expr* values)
| Set(expr* elts) | Set(expr* elts)
| ListComp(expr elt, comprehension* generators) | ListComp(expr elt, comprehension* generators)
| SetComp(expr elt, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur -- the grammar constrains where yield expressions can occur
| Yield(expr? value) | Yield(expr? value)

45
Python/Python-ast.c
View file

@ -192,6 +192,11 @@ static char *ListComp_fields[]={
"elt", "elt",
"generators", "generators",
}; };
static PyTypeObject *SetComp_type;
static char *SetComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *GeneratorExp_type; static PyTypeObject *GeneratorExp_type;
static char *GeneratorExp_fields[]={ static char *GeneratorExp_fields[]={
"elt", "elt",
@ -543,6 +548,8 @@ static int init_types(void)
if (!Set_type) return 0; if (!Set_type) return 0;
ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2); ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
if (!ListComp_type) return 0; if (!ListComp_type) return 0;
SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
if (!SetComp_type) return 0;
GeneratorExp_type = make_type("GeneratorExp", expr_type, GeneratorExp_type = make_type("GeneratorExp", expr_type,
GeneratorExp_fields, 2); GeneratorExp_fields, 2);
if (!GeneratorExp_type) return 0; if (!GeneratorExp_type) return 0;
@ -1418,6 +1425,27 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
return p; return p;
} }
expr_ty
SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!elt) {
PyErr_SetString(PyExc_ValueError,
"field elt is required for SetComp");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = SetComp_kind;
p->v.SetComp.elt = elt;
p->v.SetComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena) PyArena *arena)
@ -2416,6 +2444,21 @@ ast2obj_expr(void* _o)
goto failed; goto failed;
Py_DECREF(value); Py_DECREF(value);
break; break;
case SetComp_kind:
result = PyType_GenericNew(SetComp_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.SetComp.elt);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.SetComp.generators,
ast2obj_comprehension);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
Py_DECREF(value);
break;
case GeneratorExp_kind: case GeneratorExp_kind:
result = PyType_GenericNew(GeneratorExp_type, NULL, NULL); result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
if (!result) goto failed; if (!result) goto failed;
@ -3120,6 +3163,8 @@ init_ast(void)
if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return; if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return;
if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0) if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0)
return; return;
if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0)
return;
if (PyDict_SetItemString(d, "GeneratorExp", if (PyDict_SetItemString(d, "GeneratorExp",
(PyObject*)GeneratorExp_type) < 0) return; (PyObject*)GeneratorExp_type) < 0) return;
if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return; if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return;

366
Python/ast.c
View file

@ -27,7 +27,6 @@ static stmt_ty ast_for_stmt(struct compiling *, const node *);
static asdl_seq *ast_for_suite(struct compiling *, const node *); static asdl_seq *ast_for_suite(struct compiling *, const node *);
static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty); static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *); static expr_ty ast_for_testlist(struct compiling *, const node *);
static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);
/* Note different signature for ast_for_call */ /* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
@ -41,6 +40,10 @@ static PyObject *parsestrplus(struct compiling *, const node *n,
#define LINENO(n) ((n)->n_lineno) #define LINENO(n) ((n)->n_lineno)
#endif #endif
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
static identifier static identifier
new_identifier(const char* n, PyArena *arena) { new_identifier(const char* n, PyArena *arena) {
PyObject* id = PyString_InternFromString(n); PyObject* id = PyString_InternFromString(n);
@ -231,7 +234,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
case eval_input: { case eval_input: {
expr_ty testlist_ast; expr_ty testlist_ast;
/* XXX Why not gen_for here? */ /* XXX Why not comp_for here? */
testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
if (!testlist_ast) if (!testlist_ast)
goto error; goto error;
@ -530,19 +533,14 @@ seq_for_testlist(struct compiling *c, const node *n)
asdl_seq *seq; asdl_seq *seq;
expr_ty expression; expr_ty expression;
int i; int i;
assert(TYPE(n) == testlist assert(TYPE(n) == testlist || TYPE(n) == testlist_comp);
|| TYPE(n) == listmaker
|| TYPE(n) == testlist_gexp
|| TYPE(n) == testlist_safe
|| TYPE(n) == testlist1
);
seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!seq) if (!seq)
return NULL; return NULL;
for (i = 0; i < NCH(n); i += 2) { for (i = 0; i < NCH(n); i += 2) {
assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test); assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond);
expression = ast_for_expr(c, CHILD(n, i)); expression = ast_for_expr(c, CHILD(n, i));
if (!expression) if (!expression)
@ -1022,7 +1020,8 @@ ast_for_funcdef(struct compiling *c, const node *n)
static expr_ty static expr_ty
ast_for_lambdef(struct compiling *c, const node *n) ast_for_lambdef(struct compiling *c, const node *n)
{ {
/* lambdef: 'lambda' [varargslist] ':' test */ /* lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
arguments_ty args; arguments_ty args;
expr_ty expression; expr_ty expression;
@ -1067,190 +1066,34 @@ ast_for_ifexpr(struct compiling *c, const node *n)
c->c_arena); c->c_arena);
} }
/* XXX(nnorwitz): the listcomp and genexpr code should be refactored
so there is only a single version. Possibly for loops can also re-use
the code.
*/
/* Count the number of 'for' loop in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_fors(const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_list_for:
n_fors++;
REQ(ch, list_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
else
return n_fors;
count_list_iter:
REQ(ch, list_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == list_for)
goto count_list_for;
else if (TYPE(ch) == list_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_list_iter;
}
else
return n_fors;
}
/* Should never be reached */
PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");
return -1;
}
/* Count the number of 'if' statements in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_ifs(const node *n)
{
int n_ifs = 0;
count_list_iter:
REQ(n, list_iter);
if (TYPE(CHILD(n, 0)) == list_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, list_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
n = CHILD(n, 2);
goto count_list_iter;
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
/* listmaker: test ( list_for | (',' test)* [','] )
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_iter: list_for | list_if
list_if: 'if' test [list_iter]
testlist_safe: test [(',' test)+ [',']]
*/
expr_ty elt;
asdl_seq *listcomps;
int i, n_fors;
node *ch;
REQ(n, listmaker);
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_list_fors(n);
if (n_fors == -1)
return NULL;
listcomps = asdl_seq_new(n_fors, c->c_arena);
if (!listcomps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty lc;
asdl_seq *t;
expr_ty expression;
node *for_ch;
REQ(ch, list_for);
for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
return NULL;
expression = ast_for_testlist(c, CHILD(ch, 3));
if (!expression)
return NULL;
/* Check the # of children rather than the length of t, since
[x for x, in ... ] has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1)
lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL,
c->c_arena);
else
lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!lc)
return NULL;
if (NCH(ch) == 5) {
int j, n_ifs;
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_list_ifs(ch);
if (n_ifs == -1)
return NULL;
ifs = asdl_seq_new(n_ifs, c->c_arena);
if (!ifs)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, list_iter);
ch = CHILD(ch, 0);
REQ(ch, list_if);
asdl_seq_SET(ifs, j, ast_for_expr(c, CHILD(ch, 1)));
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
}
/* on exit, must guarantee that ch is a list_for */
if (TYPE(ch) == list_iter)
ch = CHILD(ch, 0);
lc->ifs = ifs;
}
asdl_seq_SET(listcomps, i, lc);
}
return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/* /*
Count the number of 'for' loops in a generator expression. Count the number of 'for' loops in a comprehension.
Helper for ast_for_genexp(). Helper for ast_for_comprehension().
*/ */
static int static int
count_gen_fors(const node *n) count_comp_fors(const node *n)
{ {
int n_fors = 0; int n_fors = 0;
node *ch = CHILD(n, 1); node *ch = CHILD(n, 1);
count_gen_for: count_comp_for:
n_fors++; n_fors++;
REQ(ch, gen_for); REQ(ch, comp_for);
if (NCH(ch) == 5) if (NCH(ch) == 5)
ch = CHILD(ch, 4); ch = CHILD(ch, 4);
else else
return n_fors; return n_fors;
count_gen_iter: count_comp_iter:
REQ(ch, gen_iter); REQ(ch, comp_iter);
ch = CHILD(ch, 0); ch = CHILD(ch, 0);
if (TYPE(ch) == gen_for) if (TYPE(ch) == comp_for)
goto count_gen_for; goto count_comp_for;
else if (TYPE(ch) == gen_if) { else if (TYPE(ch) == comp_if) {
if (NCH(ch) == 3) { if (NCH(ch) == 3) {
ch = CHILD(ch, 2); ch = CHILD(ch, 2);
goto count_gen_iter; goto count_comp_iter;
} }
else else
return n_fors; return n_fors;
@ -1258,26 +1101,26 @@ count_gen_fors(const node *n)
/* Should never be reached */ /* Should never be reached */
PyErr_SetString(PyExc_SystemError, PyErr_SetString(PyExc_SystemError,
"logic error in count_gen_fors"); "logic error in count_comp_fors");
return -1; return -1;
} }
/* Count the number of 'if' statements in a generator expression. /* Count the number of 'if' statements in a comprehension.
Helper for ast_for_genexp(). Helper for ast_for_comprehension().
*/ */
static int static int
count_gen_ifs(const node *n) count_comp_ifs(const node *n)
{ {
int n_ifs = 0; int n_ifs = 0;
while (1) { while (1) {
REQ(n, gen_iter); REQ(n, comp_iter);
if (TYPE(CHILD(n, 0)) == gen_for) if (TYPE(CHILD(n, 0)) == comp_for)
return n_ifs; return n_ifs;
n = CHILD(n, 0); n = CHILD(n, 0);
REQ(n, gen_if); REQ(n, comp_if);
n_ifs++; n_ifs++;
if (NCH(n) == 2) if (NCH(n) == 2)
return n_ifs; return n_ifs;
@ -1285,40 +1128,38 @@ count_gen_ifs(const node *n)
} }
} }
/* TODO(jhylton): Combine with list comprehension code? */
static expr_ty static expr_ty
ast_for_genexp(struct compiling *c, const node *n) ast_for_comprehension(struct compiling *c, const node *n, int type)
{ {
/* testlist_gexp: test ( gen_for | (',' test)* [','] ) /* testlist_comp: test ( comp_for | (',' test)* [','] )
argument: [test '='] test [gen_for] # Really [keyword '='] test */ argument: [test '='] test [comp_for] # Really [keyword '='] test */
expr_ty elt; expr_ty elt;
asdl_seq *genexps; asdl_seq *comps;
int i, n_fors; int i, n_fors;
node *ch; node *ch;
assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument));
assert(NCH(n) > 1); assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0)); elt = ast_for_expr(c, CHILD(n, 0));
if (!elt) if (!elt)
return NULL; return NULL;
n_fors = count_gen_fors(n); n_fors = count_comp_fors(n);
if (n_fors == -1) if (n_fors == -1)
return NULL; return NULL;
genexps = asdl_seq_new(n_fors, c->c_arena); comps = asdl_seq_new(n_fors, c->c_arena);
if (!genexps) if (!comps)
return NULL; return NULL;
ch = CHILD(n, 1); ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) { for (i = 0; i < n_fors; i++) {
comprehension_ty ge; comprehension_ty comp;
asdl_seq *t; asdl_seq *t;
expr_ty expression; expr_ty expression;
node *for_ch; node *for_ch;
REQ(ch, gen_for); REQ(ch, comp_for);
for_ch = CHILD(ch, 1); for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store); t = ast_for_exprlist(c, for_ch, Store);
@ -1331,14 +1172,14 @@ ast_for_genexp(struct compiling *c, const node *n)
/* Check the # of children rather than the length of t, since /* Check the # of children rather than the length of t, since
(x for x, in ...) has 1 element in t, but still requires a Tuple. */ (x for x, in ...) has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1) if (NCH(for_ch) == 1)
ge = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, comp = comprehension((expr_ty)asdl_seq_GET(t, 0), expression,
NULL, c->c_arena); NULL, c->c_arena);
else else
ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, comp = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena), c->c_arena),
expression, NULL, c->c_arena); expression, NULL, c->c_arena);
if (!ge) if (!comp)
return NULL; return NULL;
if (NCH(ch) == 5) { if (NCH(ch) == 5) {
@ -1346,7 +1187,7 @@ ast_for_genexp(struct compiling *c, const node *n)
asdl_seq *ifs; asdl_seq *ifs;
ch = CHILD(ch, 4); ch = CHILD(ch, 4);
n_ifs = count_gen_ifs(ch); n_ifs = count_comp_ifs(ch);
if (n_ifs == -1) if (n_ifs == -1)
return NULL; return NULL;
@ -1355,9 +1196,9 @@ ast_for_genexp(struct compiling *c, const node *n)
return NULL; return NULL;
for (j = 0; j < n_ifs; j++) { for (j = 0; j < n_ifs; j++) {
REQ(ch, gen_iter); REQ(ch, comp_iter);
ch = CHILD(ch, 0); ch = CHILD(ch, 0);
REQ(ch, gen_if); REQ(ch, comp_if);
expression = ast_for_expr(c, CHILD(ch, 1)); expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression) if (!expression)
@ -1366,22 +1207,52 @@ ast_for_genexp(struct compiling *c, const node *n)
if (NCH(ch) == 3) if (NCH(ch) == 3)
ch = CHILD(ch, 2); ch = CHILD(ch, 2);
} }
/* on exit, must guarantee that ch is a gen_for */ /* on exit, must guarantee that ch is a comp_for */
if (TYPE(ch) == gen_iter) if (TYPE(ch) == comp_iter)
ch = CHILD(ch, 0); ch = CHILD(ch, 0);
ge->ifs = ifs; comp->ifs = ifs;
} }
asdl_seq_SET(genexps, i, ge); asdl_seq_SET(comps, i, comp);
} }
return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena); if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_LISTCOMP)
return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
} }
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument));
return ast_for_comprehension(c, n, COMP_GENEXP);
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp));
return ast_for_comprehension(c, n, COMP_LISTCOMP);
}
static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_comprehension(c, n, COMP_SETCOMP);
}
static expr_ty static expr_ty
ast_for_atom(struct compiling *c, const node *n) ast_for_atom(struct compiling *c, const node *n)
{ {
/* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
| '{' [dictsetmaker] '}' | NAME | NUMBER | STRING+ | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
*/ */
node *ch = CHILD(n, 0); node *ch = CHILD(n, 0);
int bytesmode = 0; int bytesmode = 0;
@ -1420,18 +1291,19 @@ ast_for_atom(struct compiling *c, const node *n)
if (TYPE(ch) == yield_expr) if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch); return ast_for_expr(c, ch);
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for)) /* testlist_comp: test ( comp_for | (',' test)* [','] ) */
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for))
return ast_for_genexp(c, ch); return ast_for_genexp(c, ch);
return ast_for_testlist_gexp(c, ch); return ast_for_testlist(c, ch);
case LSQB: /* list (or list comprehension) */ case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1); ch = CHILD(n, 1);
if (TYPE(ch) == RSQB) if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, listmaker); REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch); asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts) if (!elts)
@ -1442,27 +1314,32 @@ ast_for_atom(struct compiling *c, const node *n)
else else
return ast_for_listcomp(c, ch); return ast_for_listcomp(c, ch);
case LBRACE: { case LBRACE: {
/* dictsetmaker: test ':' test (',' test ':' test)* [','] | /* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
* test (',' test)* [','] */ * test (gen_for | (',' test)* [',']) */
int i, size; int i, size;
asdl_seq *keys, *values; asdl_seq *keys, *values;
ch = CHILD(n, 1); ch = CHILD(n, 1);
if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) { if (TYPE(ch) == RBRACE) {
/* it's a set */ /* it's an empty dict */
return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
} else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
/* it's a simple set */
size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */ size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
keys = asdl_seq_new(size, c->c_arena); asdl_seq *elts = asdl_seq_new(size, c->c_arena);
if (!keys) if (!elts)
return NULL; return NULL;
for (i = 0; i < NCH(ch); i += 2) { for (i = 0; i < NCH(ch); i += 2) {
expr_ty expression; expr_ty expression;
expression = ast_for_expr(c, CHILD(ch, i)); expression = ast_for_expr(c, CHILD(ch, i));
if (!expression) if (!expression)
return NULL; return NULL;
asdl_seq_SET(keys, i / 2, expression); asdl_seq_SET(elts, i / 2, expression);
} }
return Set(keys, LINENO(n), n->n_col_offset, c->c_arena); return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
} else if (TYPE(CHILD(ch, 1)) == comp_for) {
/* it's a set comprehension */
return ast_for_setcomp(c, ch);
} else { } else {
/* it's a dict */ /* it's a dict */
size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */ size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
@ -1790,6 +1667,7 @@ ast_for_expr(struct compiling *c, const node *n)
{ {
/* handle the full range of simple expressions /* handle the full range of simple expressions
test: or_test ['if' or_test 'else' test] | lambdef test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
or_test: and_test ('or' and_test)* or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)* and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison not_test: 'not' not_test | comparison
@ -1802,15 +1680,6 @@ ast_for_expr(struct compiling *c, const node *n)
term: factor (('*'|'/'|'%'|'//') factor)* term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power factor: ('+'|'-'|'~') factor | power
power: atom trailer* ('**' factor)* power: atom trailer* ('**' factor)*
As well as modified versions that exist for backward compatibility,
to explicitly allow:
[ x for x in lambda: 0, lambda: 1 ]
(which would be ambiguous without these extra rules)
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [vararglist] ':' old_test
*/ */
asdl_seq *seq; asdl_seq *seq;
@ -1819,9 +1688,9 @@ ast_for_expr(struct compiling *c, const node *n)
loop: loop:
switch (TYPE(n)) { switch (TYPE(n)) {
case test: case test:
case old_test: case test_nocond:
if (TYPE(CHILD(n, 0)) == lambdef || if (TYPE(CHILD(n, 0)) == lambdef ||
TYPE(CHILD(n, 0)) == old_lambdef) TYPE(CHILD(n, 0)) == lambdef_nocond)
return ast_for_lambdef(c, CHILD(n, 0)); return ast_for_lambdef(c, CHILD(n, 0));
else if (NCH(n) > 1) else if (NCH(n) > 1)
return ast_for_ifexpr(c, n); return ast_for_ifexpr(c, n);
@ -1947,7 +1816,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
/* /*
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
| '**' test) | '**' test)
argument: [test '='] test [gen_for] # Really [keyword '='] test argument: [test '='] test [comp_for] # Really [keyword '='] test
*/ */
int i, nargs, nkeywords, ngens; int i, nargs, nkeywords, ngens;
@ -1965,7 +1834,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
if (TYPE(ch) == argument) { if (TYPE(ch) == argument) {
if (NCH(ch) == 1) if (NCH(ch) == 1)
nargs++; nargs++;
else if (TYPE(CHILD(ch, 1)) == gen_for) else if (TYPE(CHILD(ch, 1)) == comp_for)
ngens++; ngens++;
else else
nkeywords++; nkeywords++;
@ -2005,7 +1874,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
return NULL; return NULL;
asdl_seq_SET(args, nargs++, e); asdl_seq_SET(args, nargs++, e);
} }
else if (TYPE(CHILD(ch, 1)) == gen_for) { else if (TYPE(CHILD(ch, 1)) == comp_for) {
e = ast_for_genexp(c, ch); e = ast_for_genexp(c, ch);
if (!e) if (!e)
return NULL; return NULL;
@ -2057,18 +1926,16 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
static expr_ty static expr_ty
ast_for_testlist(struct compiling *c, const node* n) ast_for_testlist(struct compiling *c, const node* n)
{ {
/* testlist_gexp: test (',' test)* [','] */ /* testlist_comp: test (comp_for | (',' test)* [',']) */
/* testlist: test (',' test)* [','] */ /* testlist: test (',' test)* [','] */
/* testlist_safe: test (',' test)+ [','] */
/* testlist1: test (',' test)* */ /* testlist1: test (',' test)* */
assert(NCH(n) > 0); assert(NCH(n) > 0);
if (TYPE(n) == testlist_gexp) { if (TYPE(n) == testlist_comp) {
if (NCH(n) > 1) if (NCH(n) > 1)
assert(TYPE(CHILD(n, 1)) != gen_for); assert(TYPE(CHILD(n, 1)) != comp_for);
} }
else { else {
assert(TYPE(n) == testlist || assert(TYPE(n) == testlist ||
TYPE(n) == testlist_safe ||
TYPE(n) == testlist1); TYPE(n) == testlist1);
} }
if (NCH(n) == 1) if (NCH(n) == 1)
@ -2081,17 +1948,6 @@ ast_for_testlist(struct compiling *c, const node* n)
} }
} }
static expr_ty
ast_for_testlist_gexp(struct compiling *c, const node* n)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] ) */
/* argument: test [ gen_for ] */
assert(TYPE(n) == testlist_gexp || TYPE(n) == argument);
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for)
return ast_for_genexp(c, n);
return ast_for_testlist(c, n);
}
static stmt_ty static stmt_ty
ast_for_expr_stmt(struct compiling *c, const node *n) ast_for_expr_stmt(struct compiling *c, const node *n)
{ {

12
Python/ceval.c
View file

@ -1241,6 +1241,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
} }
break; break;
case SET_ADD:
w = POP();
v = POP();
err = PySet_Add(v, w);
Py_DECREF(v);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
continue;
}
break;
case INPLACE_POWER: case INPLACE_POWER:
w = POP(); w = POP();
v = TOP(); v = TOP();

295
Python/compile.c
View file

@ -39,6 +39,10 @@ int Py_OptimizeFlag = 0;
#define DEFAULT_CODE_SIZE 128 #define DEFAULT_CODE_SIZE 128
#define DEFAULT_LNOTAB_SIZE 16 #define DEFAULT_LNOTAB_SIZE 16
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
struct instr { struct instr {
unsigned i_jabs : 1; unsigned i_jabs : 1;
unsigned i_jrel : 1; unsigned i_jrel : 1;
@ -360,7 +364,7 @@ dictbytype(PyObject *src, int scope_type, int flag, int offset)
while (PyDict_Next(src, &pos, &k, &v)) { while (PyDict_Next(src, &pos, &k, &v)) {
/* XXX this should probably be a macro in symtable.h */ /* XXX this should probably be a macro in symtable.h */
assert(PyInt_Check(v)); assert(PyInt_Check(v));
scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; scope = (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
if (scope == scope_type || PyInt_AS_LONG(v) & flag) { if (scope == scope_type || PyInt_AS_LONG(v) & flag) {
PyObject *tuple, *item = PyInt_FromLong(i); PyObject *tuple, *item = PyInt_FromLong(i);
@ -673,6 +677,7 @@ opcode_stack_effect(int opcode, int oparg)
case UNARY_INVERT: case UNARY_INVERT:
return 0; return 0;
case SET_ADD:
case LIST_APPEND: case LIST_APPEND:
return -2; return -2;
@ -2724,15 +2729,31 @@ compiler_call_helper(struct compiler *c,
return 1; return 1;
} }
/* List and set comprehensions and generator expressions work by creating a
nested function to perform the actual iteration. This means that the
iteration variables don't leak into the current scope.
The defined function is called immediately following its definition, with the
result of that call being the result of the expression.
The LC/SC version returns the populated container, while the GE version is
flagged in symtable.c as a generator, so it returns the generator object
when the function is called.
This code *knows* that the loop cannot contain break, continue, or return,
so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
Possible cleanups:
- iterate over the generator sequence instead of using recursion
*/
static int static int
compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, compiler_comprehension_generator(struct compiler *c, PyObject *tmpname,
asdl_seq *generators, int gen_index, asdl_seq *generators, int gen_index,
expr_ty elt) expr_ty elt, int type)
{ {
/* generate code for the iterator, then each of the ifs, /* generate code for the iterator, then each of the ifs,
and then write to the element */ and then write to the element */
comprehension_ty l; comprehension_ty gen;
basicblock *start, *anchor, *skip, *if_cleanup; basicblock *start, *anchor, *skip, *if_cleanup;
int i, n; int i, n;
@ -2742,104 +2763,11 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
anchor = compiler_new_block(c); anchor = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL || if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL) anchor == NULL)
return 0;
l = (comprehension_ty)asdl_seq_GET(generators, gen_index);
VISIT(c, expr, l->iter);
ADDOP(c, GET_ITER);
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, l->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(l->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i);
VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
ADDOP(c, POP_TOP);
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_listcomp_generator(c, tmpname,
generators, gen_index, elt))
return 0; return 0;
/* only append after the last for generator */ gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
if (gen_index >= asdl_seq_LEN(generators)) {
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
compiler_use_next_block(c, skip);
}
for (i = 0; i < n; i++) {
ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0)
compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP);
}
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
/* delete the temporary list name added to locals */
if (gen_index == 1)
if (!compiler_nameop(c, tmpname, Del))
return 0;
return 1;
}
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
identifier tmp;
int rc = 0;
asdl_seq *generators = e->v.ListComp.generators;
assert(e->kind == ListComp_kind);
tmp = compiler_new_tmpname(c);
if (!tmp)
return 0;
ADDOP_I(c, BUILD_LIST, 0);
ADDOP(c, DUP_TOP);
if (compiler_nameop(c, tmp, Store))
rc = compiler_listcomp_generator(c, tmp, generators, 0,
e->v.ListComp.elt);
Py_DECREF(tmp);
return rc;
}
static int
compiler_genexp_generator(struct compiler *c,
asdl_seq *generators, int gen_index,
expr_ty elt)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty ge;
basicblock *start, *anchor, *skip, *if_cleanup, *end;
int i, n;
start = compiler_new_block(c);
skip = compiler_new_block(c);
if_cleanup = compiler_new_block(c);
anchor = compiler_new_block(c);
end = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL || end == NULL)
return 0;
ge = (comprehension_ty)asdl_seq_GET(generators, gen_index);
ADDOP_JREL(c, SETUP_LOOP, end);
if (!compiler_push_fblock(c, LOOP, start))
return 0;
if (gen_index == 0) { if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */ /* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1; c->u->u_argcount = 1;
@ -2847,18 +2775,18 @@ compiler_genexp_generator(struct compiler *c,
} }
else { else {
/* Sub-iter - calculate on the fly */ /* Sub-iter - calculate on the fly */
VISIT(c, expr, ge->iter); VISIT(c, expr, gen->iter);
ADDOP(c, GET_ITER); ADDOP(c, GET_ITER);
} }
compiler_use_next_block(c, start); compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor); ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c); NEXT_BLOCK(c);
VISIT(c, expr, ge->target); VISIT(c, expr, gen->target);
/* XXX this needs to be cleaned up...a lot! */ /* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(ge->ifs); n = asdl_seq_LEN(gen->ifs);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i); expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
VISIT(c, expr, e); VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c); NEXT_BLOCK(c);
@ -2866,14 +2794,35 @@ compiler_genexp_generator(struct compiler *c,
} }
if (++gen_index < asdl_seq_LEN(generators)) if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_genexp_generator(c, generators, gen_index, elt)) if (!compiler_comprehension_generator(c, tmpname,
return 0; generators, gen_index,
elt, type))
return 0;
/* only append after the last 'for' generator */ /* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) { if (gen_index >= asdl_seq_LEN(generators)) {
VISIT(c, expr, elt); /* comprehension specific code */
ADDOP(c, YIELD_VALUE); switch (type) {
ADDOP(c, POP_TOP); case COMP_GENEXP:
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
break;
case COMP_LISTCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
break;
case COMP_SETCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, SET_ADD);
break;
default:
return 0;
}
compiler_use_next_block(c, skip); compiler_use_next_block(c, skip);
} }
@ -2881,52 +2830,116 @@ compiler_genexp_generator(struct compiler *c,
ADDOP_I(c, JUMP_FORWARD, 1); ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0) if (i == 0)
compiler_use_next_block(c, if_cleanup); compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP); ADDOP(c, POP_TOP);
} }
ADDOP_JABS(c, JUMP_ABSOLUTE, start); ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor); compiler_use_next_block(c, anchor);
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, LOOP, start);
compiler_use_next_block(c, end);
return 1; return 1;
} }
static int
compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
asdl_seq *generators, expr_ty elt)
{
PyCodeObject *co = NULL;
identifier tmp = NULL;
expr_ty outermost_iter;
outermost_iter = ((comprehension_ty)
asdl_seq_GET(generators, 0))->iter;
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
goto error;
if (type != COMP_GENEXP) {
tmp = compiler_new_tmpname(c);
if (!tmp)
goto error_in_scope;
ADDOP_I(c, (type == COMP_LISTCOMP ?
BUILD_LIST : BUILD_SET), 0);
ADDOP(c, DUP_TOP);
if (!compiler_nameop(c, tmp, Store))
goto error_in_scope;
}
if (!compiler_comprehension_generator(c, tmp, generators, 0, elt, type))
goto error_in_scope;
if (type != COMP_GENEXP) {
ADDOP(c, RETURN_VALUE);
}
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
goto error;
if (!compiler_make_closure(c, co, 0))
goto error;
Py_DECREF(co);
Py_XDECREF(tmp);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
error_in_scope:
compiler_exit_scope(c);
error:
Py_XDECREF(co);
Py_XDECREF(tmp);
return 0;
}
static int static int
compiler_genexp(struct compiler *c, expr_ty e) compiler_genexp(struct compiler *c, expr_ty e)
{ {
static identifier name; static identifier name;
PyCodeObject *co;
expr_ty outermost_iter = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators,
0)))->iter;
if (!name) { if (!name) {
name = PyString_FromString("<genexpr>"); name = PyString_FromString("<genexp>");
if (!name) if (!name)
return 0; return 0;
} }
assert(e->kind == GeneratorExp_kind);
if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) return compiler_comprehension(c, e, COMP_GENEXP, name,
return 0; e->v.GeneratorExp.generators,
compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0, e->v.GeneratorExp.elt);
e->v.GeneratorExp.elt);
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
return 0;
compiler_make_closure(c, co, 0);
Py_DECREF(co);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
} }
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<listcomp>");
if (!name)
return 0;
}
assert(e->kind == ListComp_kind);
return compiler_comprehension(c, e, COMP_LISTCOMP, name,
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
compiler_setcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<setcomp>");
if (!name)
return 0;
}
assert(e->kind == SetComp_kind);
return compiler_comprehension(c, e, COMP_SETCOMP, name,
e->v.SetComp.generators,
e->v.SetComp.elt);
}
static int static int
compiler_visit_keyword(struct compiler *c, keyword_ty k) compiler_visit_keyword(struct compiler *c, keyword_ty k)
{ {
@ -3145,10 +3158,12 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
VISIT_SEQ(c, expr, e->v.Set.elts); VISIT_SEQ(c, expr, e->v.Set.elts);
ADDOP_I(c, BUILD_SET, n); ADDOP_I(c, BUILD_SET, n);
break; break;
case ListComp_kind:
return compiler_listcomp(c, e);
case GeneratorExp_kind: case GeneratorExp_kind:
return compiler_genexp(c, e); return compiler_genexp(c, e);
case ListComp_kind:
return compiler_listcomp(c, e);
case SetComp_kind:
return compiler_setcomp(c, e);
case Yield_kind: case Yield_kind:
if (c->u->u_ste->ste_type != FunctionBlock) if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function"); return compiler_error(c, "'yield' outside function");

986
Python/graminit.c
View file

File diff suppressed because it is too large Load diff

347
Python/symtable.c
View file

@ -76,7 +76,7 @@ PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_generator = 0; ste->ste_generator = 0;
ste->ste_returns_value = 0; ste->ste_returns_value = 0;
if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
goto fail; goto fail;
return ste; return ste;
@ -172,6 +172,8 @@ static int symtable_exit_block(struct symtable *st, void *ast);
static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s);
static int symtable_visit_genexp(struct symtable *st, expr_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s);
static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_arguments(struct symtable *st, arguments_ty);
static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_alias(struct symtable *st, alias_ty);
@ -186,7 +188,8 @@ static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s); static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
static identifier top = NULL, lambda = NULL, genexpr = NULL; static identifier top = NULL, lambda = NULL, genexpr = NULL,
listcomp = NULL, setcomp = NULL;
#define GET_IDENTIFIER(VAR) \ #define GET_IDENTIFIER(VAR) \
((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR)))
@ -204,14 +207,13 @@ symtable_new(void)
return NULL; return NULL;
st->st_filename = NULL; st->st_filename = NULL;
st->st_symbols = NULL; st->st_blocks = NULL;
if ((st->st_stack = PyList_New(0)) == NULL) if ((st->st_stack = PyList_New(0)) == NULL)
goto fail; goto fail;
if ((st->st_symbols = PyDict_New()) == NULL) if ((st->st_blocks = PyDict_New()) == NULL)
goto fail; goto fail;
st->st_cur = NULL; st->st_cur = NULL;
st->st_tmpname = 0;
st->st_private = NULL; st->st_private = NULL;
return st; return st;
fail: fail:
@ -230,6 +232,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
return st; return st;
st->st_filename = filename; st->st_filename = filename;
st->st_future = future; st->st_future = future;
/* Make the initial symbol information gathering pass */
if (!GET_IDENTIFIER(top) || if (!GET_IDENTIFIER(top) ||
!symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) { !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) {
PySymtable_Free(st); PySymtable_Free(st);
@ -238,7 +241,6 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
st->st_top = st->st_cur; st->st_top = st->st_cur;
st->st_cur->ste_unoptimized = OPT_TOPLEVEL; st->st_cur->ste_unoptimized = OPT_TOPLEVEL;
/* Any other top-level initialization? */
switch (mod->kind) { switch (mod->kind) {
case Module_kind: case Module_kind:
seq = mod->v.Module.body; seq = mod->v.Module.body;
@ -267,6 +269,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
PySymtable_Free(st); PySymtable_Free(st);
return NULL; return NULL;
} }
/* Make the second symbol analysis pass */
if (symtable_analyze(st)) if (symtable_analyze(st))
return st; return st;
PySymtable_Free(st); PySymtable_Free(st);
@ -280,7 +283,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
void void
PySymtable_Free(struct symtable *st) PySymtable_Free(struct symtable *st)
{ {
Py_XDECREF(st->st_symbols); Py_XDECREF(st->st_blocks);
Py_XDECREF(st->st_stack); Py_XDECREF(st->st_stack);
PyMem_Free((void *)st); PyMem_Free((void *)st);
} }
@ -293,7 +296,7 @@ PySymtable_Lookup(struct symtable *st, void *key)
k = PyLong_FromVoidPtr(key); k = PyLong_FromVoidPtr(key);
if (k == NULL) if (k == NULL)
return NULL; return NULL;
v = PyDict_GetItem(st->st_symbols, k); v = PyDict_GetItem(st->st_blocks, k);
if (v) { if (v) {
assert(PySTEntry_Check(v)); assert(PySTEntry_Check(v));
Py_INCREF(v); Py_INCREF(v);
@ -314,7 +317,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
if (!v) if (!v)
return 0; return 0;
assert(PyInt_Check(v)); assert(PyInt_Check(v));
return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; return (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
} }
@ -325,7 +328,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
it determines which local variables are cell variables; they provide it determines which local variables are cell variables; they provide
bindings that are used for free variables in enclosed blocks. bindings that are used for free variables in enclosed blocks.
There are also two kinds of free variables, implicit and explicit. An There are also two kinds of global variables, implicit and explicit. An
explicit global is declared with the global statement. An implicit explicit global is declared with the global statement. An implicit
global is a free variable for which the compiler has found no binding global is a free variable for which the compiler has found no binding
in an enclosing function scope. The implicit global is either a global in an enclosing function scope. The implicit global is either a global
@ -337,24 +340,30 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
TODO(jhylton): Discuss nonlocal TODO(jhylton): Discuss nonlocal
The symbol table requires two passes to determine the scope of each name. The symbol table requires two passes to determine the scope of each name.
The first pass collects raw facts from the AST: the name is a parameter The first pass collects raw facts from the AST via the symtable_visit_*
here, the name is used by not defined here, etc. The second pass analyzes functions: the name is a parameter here, the name is used but not defined
these facts during a pass over the PySTEntryObjects created during pass 1. here, etc. The second pass analyzes these facts during a pass over the
PySTEntryObjects created during pass 1.
When a function is entered during the second pass, the parent passes When a function is entered during the second pass, the parent passes
the set of all name bindings visible to its children. These bindings the set of all name bindings visible to its children. These bindings
are used to determine if the variable is free or an implicit global. are used to determine if non-local variables are free or implicit globals.
After doing the local analysis, it analyzes each of its child blocks After doing the local analysis, it analyzes each of its child blocks
using an updated set of name bindings. using an updated set of name bindings.
The children update the free variable set. If a local variable is free The children update the free variable set. If a local variable is added to
in a child, the variable is marked as a cell. The current function must the free variable set by the child, the variable is marked as a cell. The
provide runtime storage for the variable that may outlive the function's function object being defined must provide runtime storage for the variable
frame. Cell variables are removed from the free set before the analyze that may outlive the function's frame. Cell variables are removed from the
function returns to its parent. free set before the analyze function returns to its parent.
The sets of bound and free variables are implemented as dictionaries During analysis, the names are:
mapping strings to None. symbols: dict mapping from symbol names to flag values (including offset scope values)
scopes: dict mapping from symbol names to scope values (no offset)
local: set of all symbol names local to the current scope
bound: set of all symbol names local to a containing function scope
free: set of all symbol names referenced but not bound in child scopes
global: set of all symbol names explicitly declared as global
*/ */
#define SET_SCOPE(DICT, NAME, I) { \ #define SET_SCOPE(DICT, NAME, I) { \
@ -375,14 +384,14 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
*/ */
static int static int
analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
PyObject *bound, PyObject *local, PyObject *free, PyObject *bound, PyObject *local, PyObject *free,
PyObject *global) PyObject *global)
{ {
if (flags & DEF_GLOBAL) { if (flags & DEF_GLOBAL) {
if (flags & DEF_PARAM) { if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError, PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and global", "name '%s' is parameter and global",
PyString_AS_STRING(name)); PyString_AS_STRING(name));
return 0; return 0;
} }
@ -392,41 +401,37 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
PyString_AS_STRING(name)); PyString_AS_STRING(name));
return 0; return 0;
} }
SET_SCOPE(dict, name, GLOBAL_EXPLICIT); SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
if (PyDict_SetItem(global, name, Py_None) < 0) if (PySet_Add(global, name) < 0)
return 0;
if (bound && (PySet_Discard(bound, name) < 0))
return 0; return 0;
if (bound && PyDict_GetItem(bound, name)) {
if (PyDict_DelItem(bound, name) < 0)
return 0;
}
return 1; return 1;
} }
if (flags & DEF_NONLOCAL) { if (flags & DEF_NONLOCAL) {
if (flags & DEF_PARAM) { if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError, PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and nonlocal", "name '%s' is parameter and nonlocal",
PyString_AS_STRING(name)); PyString_AS_STRING(name));
return 0; return 0;
} }
if (!PyDict_GetItem(bound, name)) { if (!PySet_Contains(bound, name)) {
PyErr_Format(PyExc_SyntaxError, PyErr_Format(PyExc_SyntaxError,
"no binding for nonlocal '%s' found", "no binding for nonlocal '%s' found",
PyString_AS_STRING(name)); PyString_AS_STRING(name));
return 0; return 0;
} }
SET_SCOPE(dict, name, FREE); SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1; ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0; return PySet_Add(free, name) >= 0;
} }
if (flags & DEF_BOUND) { if (flags & DEF_BOUND) {
SET_SCOPE(dict, name, LOCAL); SET_SCOPE(scopes, name, LOCAL);
if (PyDict_SetItem(local, name, Py_None) < 0) if (PySet_Add(local, name) < 0)
return 0;
if (PySet_Discard(global, name) < 0)
return 0; return 0;
if (PyDict_GetItem(global, name)) {
if (PyDict_DelItem(global, name) < 0)
return 0;
}
return 1; return 1;
} }
/* If an enclosing block has a binding for this name, it /* If an enclosing block has a binding for this name, it
@ -434,21 +439,21 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
Note that having a non-NULL bound implies that the block Note that having a non-NULL bound implies that the block
is nested. is nested.
*/ */
if (bound && PyDict_GetItem(bound, name)) { if (bound && PySet_Contains(bound, name)) {
SET_SCOPE(dict, name, FREE); SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1; ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0; return PySet_Add(free, name) >= 0;
} }
/* If a parent has a global statement, then call it global /* If a parent has a global statement, then call it global
explicit? It could also be global implicit. explicit? It could also be global implicit.
*/ */
if (global && PyDict_GetItem(global, name)) { if (global && PySet_Contains(global, name)) {
SET_SCOPE(dict, name, GLOBAL_EXPLICIT); SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
return 1; return 1;
} }
if (ste->ste_nested) if (ste->ste_nested)
ste->ste_free = 1; ste->ste_free = 1;
SET_SCOPE(dict, name, GLOBAL_IMPLICIT); SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
return 1; return 1;
} }
@ -463,35 +468,35 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
*/ */
static int static int
analyze_cells(PyObject *scope, PyObject *free) analyze_cells(PyObject *scopes, PyObject *free)
{ {
PyObject *name, *v, *w; PyObject *name, *v, *v_cell;
int success = 0; int success = 0;
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
w = PyInt_FromLong(CELL); v_cell = PyInt_FromLong(CELL);
if (!w) if (!v_cell)
return 0; return 0;
while (PyDict_Next(scope, &pos, &name, &v)) { while (PyDict_Next(scopes, &pos, &name, &v)) {
long flags; long scope;
assert(PyInt_Check(v)); assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v); scope = PyInt_AS_LONG(v);
if (flags != LOCAL) if (scope != LOCAL)
continue; continue;
if (!PyDict_GetItem(free, name)) if (!PySet_Contains(free, name))
continue; continue;
/* Replace LOCAL with CELL for this name, and remove /* Replace LOCAL with CELL for this name, and remove
from free. It is safe to replace the value of name from free. It is safe to replace the value of name
in the dict, because it will not cause a resize. in the dict, because it will not cause a resize.
*/ */
if (PyDict_SetItem(scope, name, w) < 0) if (PyDict_SetItem(scopes, name, v_cell) < 0)
goto error; goto error;
if (!PyDict_DelItem(free, name) < 0) if (PySet_Discard(free, name) < 0)
goto error; goto error;
} }
success = 1; success = 1;
error: error:
Py_DECREF(w); Py_DECREF(v_cell);
return success; return success;
} }
@ -526,77 +531,91 @@ check_unoptimized(const PySTEntryObject* ste) {
return 0; return 0;
} }
/* Enter the final scope information into the st_symbols dict. /* Enter the final scope information into the ste_symbols dict.
* *
* All arguments are dicts. Modifies symbols, others are read-only. * All arguments are dicts. Modifies symbols, others are read-only.
*/ */
static int static int
update_symbols(PyObject *symbols, PyObject *scope, update_symbols(PyObject *symbols, PyObject *scopes,
PyObject *bound, PyObject *free, int classflag) PyObject *bound, PyObject *free, int classflag)
{ {
PyObject *name, *v, *u, *w, *free_value = NULL; PyObject *name = NULL, *itr = NULL;
PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
/* Update scope information for all symbols in this scope */
while (PyDict_Next(symbols, &pos, &name, &v)) { while (PyDict_Next(symbols, &pos, &name, &v)) {
long i, flags; long scope, flags;
assert(PyInt_Check(v)); assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v); flags = PyInt_AS_LONG(v);
w = PyDict_GetItem(scope, name); v_scope = PyDict_GetItem(scopes, name);
assert(w && PyInt_Check(w)); assert(v_scope && PyInt_Check(v_scope));
i = PyInt_AS_LONG(w); scope = PyInt_AS_LONG(v_scope);
flags |= (i << SCOPE_OFF); flags |= (scope << SCOPE_OFFSET);
u = PyInt_FromLong(flags); v_new = PyInt_FromLong(flags);
if (!u) if (!v_new)
return 0; return 0;
if (PyDict_SetItem(symbols, name, u) < 0) { if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(u); Py_DECREF(v_new);
return 0; return 0;
} }
Py_DECREF(u); Py_DECREF(v_new);
} }
free_value = PyInt_FromLong(FREE << SCOPE_OFF); /* Record not yet resolved free variables from children (if any) */
if (!free_value) v_free = PyInt_FromLong(FREE << SCOPE_OFFSET);
if (!v_free)
return 0; return 0;
/* add a free variable when it's only use is for creating a closure */ itr = PyObject_GetIter(free);
pos = 0; if (!itr)
while (PyDict_Next(free, &pos, &name, &v)) { goto error;
PyObject *o = PyDict_GetItem(symbols, name);
if (o) { while ((name = PyIter_Next(itr))) {
/* It could be a free variable in a method of v = PyDict_GetItem(symbols, name);
/* Handle symbol that already exists in this scope */
if (v) {
/* Handle a free variable in a method of
the class that has the same name as a local the class that has the same name as a local
or global in the class scope. or global in the class scope.
*/ */
if (classflag && if (classflag &&
PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { PyInt_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; long flags = PyInt_AS_LONG(v) | DEF_FREE_CLASS;
o = PyInt_FromLong(i); v_new = PyInt_FromLong(flags);
if (!o) { if (!v_new) {
Py_DECREF(free_value); goto error;
return 0;
} }
if (PyDict_SetItem(symbols, name, o) < 0) { if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(o); Py_DECREF(v_new);
Py_DECREF(free_value); goto error;
return 0;
} }
Py_DECREF(o); Py_DECREF(v_new);
} }
/* else it's not free, probably a cell */ /* It's a cell, or already a free variable in this scope */
Py_DECREF(name);
continue; continue;
} }
if (!PyDict_GetItem(bound, name)) /* Handle global symbol */
if (!PySet_Contains(bound, name)) {
Py_DECREF(name);
continue; /* it's a global */ continue; /* it's a global */
if (PyDict_SetItem(symbols, name, free_value) < 0) {
Py_DECREF(free_value);
return 0;
} }
/* Propagate new free symbol up the lexical stack */
if (PyDict_SetItem(symbols, name, v_free) < 0) {
goto error;
}
Py_DECREF(name);
} }
Py_DECREF(free_value); Py_DECREF(itr);
Py_DECREF(v_free);
return 1; return 1;
error:
Py_XDECREF(v_free);
Py_XDECREF(itr);
Py_XDECREF(name);
return 0;
} }
/* Make final symbol table decisions for block of ste. /* Make final symbol table decisions for block of ste.
@ -611,59 +630,74 @@ static int
analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
PyObject *global) PyObject *global)
{ {
PyObject *name, *v, *local = NULL, *scope = NULL, *newbound = NULL; PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
PyObject *newglobal = NULL, *newfree = NULL; PyObject *newglobal = NULL, *newfree = NULL;
int i, success = 0; int i, success = 0;
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
local = PyDict_New(); scopes = PyDict_New();
if (!scopes)
goto error;
local = PySet_New(NULL);
if (!local) if (!local)
goto error; goto error;
scope = PyDict_New(); newglobal = PySet_New(NULL);
if (!scope)
goto error;
newglobal = PyDict_New();
if (!newglobal) if (!newglobal)
goto error; goto error;
newfree = PyDict_New(); newfree = PySet_New(NULL);
if (!newfree) if (!newfree)
goto error; goto error;
newbound = PyDict_New(); newbound = PySet_New(NULL);
if (!newbound) if (!newbound)
goto error; goto error;
/* Class namespace has no effect on names visible in
nested functions, so populate the global and bound
sets to be passed to child blocks before analyzing
this one.
*/
if (ste->ste_type == ClassBlock) { if (ste->ste_type == ClassBlock) {
/* make a copy of globals before calling analyze_name(), /* Pass down previously bound symbols */
because global statements in the class have no effect if (bound) {
on nested functions. if (!PyNumber_InPlaceOr(newbound, bound))
*/
if (PyDict_Update(newglobal, global) < 0)
goto error;
if (bound)
if (PyDict_Update(newbound, bound) < 0)
goto error; goto error;
Py_DECREF(newbound);
}
/* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error;
Py_DECREF(newglobal);
} }
/* Analyze symbols in current scope */
assert(PySTEntry_Check(ste)); assert(PySTEntry_Check(ste));
assert(PyDict_Check(ste->ste_symbols)); assert(PyDict_Check(ste->ste_symbols));
while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
long flags = PyInt_AS_LONG(v); long flags = PyInt_AS_LONG(v);
if (!analyze_name(ste, scope, name, flags, bound, local, free, if (!analyze_name(ste, scopes, name, flags, bound, local, free,
global)) global))
goto error; goto error;
} }
/* Populate global and bound sets to be passed to children.
*/
if (ste->ste_type != ClassBlock) { if (ste->ste_type != ClassBlock) {
/* Add function locals to bound set */
if (ste->ste_type == FunctionBlock) { if (ste->ste_type == FunctionBlock) {
if (PyDict_Update(newbound, local) < 0) if (!PyNumber_InPlaceOr(newbound, local))
goto error; goto error;
Py_DECREF(newbound);
} }
/* Pass down previously bound symbols */
if (bound) { if (bound) {
if (PyDict_Update(newbound, bound) < 0) if (!PyNumber_InPlaceOr(newbound, bound))
goto error; goto error;
Py_DECREF(newbound);
} }
if (PyDict_Update(newglobal, global) < 0) /* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error; goto error;
Py_DECREF(newglobal);
} }
/* Recursively call analyze_block() on each child block */ /* Recursively call analyze_block() on each child block */
@ -674,24 +708,28 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
entry = (PySTEntryObject*)c; entry = (PySTEntryObject*)c;
if (!analyze_block(entry, newbound, newfree, newglobal)) if (!analyze_block(entry, newbound, newfree, newglobal))
goto error; goto error;
/* Check if any children have free variables */
if (entry->ste_free || entry->ste_child_free) if (entry->ste_free || entry->ste_child_free)
ste->ste_child_free = 1; ste->ste_child_free = 1;
} }
if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) /* Check if any local variables need to be converted to cell variables */
if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
goto error; goto error;
if (!update_symbols(ste->ste_symbols, scope, bound, newfree, /* Records the results of the analysis in the symbol table entry */
if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
ste->ste_type == ClassBlock)) ste->ste_type == ClassBlock))
goto error; goto error;
if (!check_unoptimized(ste)) if (!check_unoptimized(ste))
goto error; goto error;
if (PyDict_Update(free, newfree) < 0) if (!PyNumber_InPlaceOr(free, newfree))
goto error; goto error;
Py_DECREF(free);
success = 1; success = 1;
error: error:
Py_XDECREF(scopes);
Py_XDECREF(local); Py_XDECREF(local);
Py_XDECREF(scope);
Py_XDECREF(newbound); Py_XDECREF(newbound);
Py_XDECREF(newglobal); Py_XDECREF(newglobal);
Py_XDECREF(newfree); Py_XDECREF(newfree);
@ -706,10 +744,10 @@ symtable_analyze(struct symtable *st)
PyObject *free, *global; PyObject *free, *global;
int r; int r;
free = PyDict_New(); free = PySet_New(NULL);
if (!free) if (!free)
return 0; return 0;
global = PyDict_New(); global = PySet_New(NULL);
if (!global) { if (!global) {
Py_DECREF(free); Py_DECREF(free);
return 0; return 0;
@ -1200,16 +1238,18 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
case Set_kind: case Set_kind:
VISIT_SEQ(st, expr, e->v.Set.elts); VISIT_SEQ(st, expr, e->v.Set.elts);
break; break;
case ListComp_kind:
if (!symtable_new_tmpname(st))
return 0;
VISIT(st, expr, e->v.ListComp.elt);
VISIT_SEQ(st, comprehension, e->v.ListComp.generators);
break;
case GeneratorExp_kind: case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e)) if (!symtable_visit_genexp(st, e))
return 0; return 0;
break; break;
case ListComp_kind:
if (!symtable_visit_listcomp(st, e))
return 0;
break;
case SetComp_kind:
if (!symtable_visit_setcomp(st, e))
return 0;
break;
case Yield_kind: case Yield_kind:
if (e->v.Yield.value) if (e->v.Yield.value)
VISIT(st, expr, e->v.Yield.value); VISIT(st, expr, e->v.Yield.value);
@ -1479,27 +1519,60 @@ symtable_visit_slice(struct symtable *st, slice_ty s)
} }
static int static int
symtable_visit_genexp(struct symtable *st, expr_ty e) symtable_handle_comprehension(struct symtable *st, expr_ty e,
identifier scope_name,
asdl_seq *generators, expr_ty elt)
{ {
int is_generator = (e->kind == GeneratorExp_kind);
int needs_tmp = !is_generator;
comprehension_ty outermost = ((comprehension_ty) comprehension_ty outermost = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators, 0))); asdl_seq_GET(generators, 0));
/* Outermost iterator is evaluated in current scope */ /* Outermost iterator is evaluated in current scope */
VISIT(st, expr, outermost->iter); VISIT(st, expr, outermost->iter);
/* Create generator scope for the rest */ /* Create comprehension scope for the rest */
if (!GET_IDENTIFIER(genexpr) || if (!scope_name ||
!symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, 0)) { !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
return 0; return 0;
} }
st->st_cur->ste_generator = 1; st->st_cur->ste_generator = is_generator;
/* Outermost iter is received as an argument */ /* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) { if (!symtable_implicit_arg(st, 0)) {
symtable_exit_block(st, (void *)e); symtable_exit_block(st, (void *)e);
return 0; return 0;
} }
/* Allocate temporary name if needed */
if (needs_tmp && !symtable_new_tmpname(st)) {
symtable_exit_block(st, (void *)e);
return 0;
}
VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e);
VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e);
VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension,
e->v.GeneratorExp.generators, 1, (void*)e); generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e); VISIT_IN_BLOCK(st, expr, elt, (void*)e);
return symtable_exit_block(st, (void *)e); return symtable_exit_block(st, (void *)e);
} }
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt);
}
static int
symtable_visit_listcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp),
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
symtable_visit_setcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
e->v.SetComp.generators,
e->v.SetComp.elt);
}

4
setup.py
View file

@ -1175,8 +1175,8 @@ class PyBuildExt(build_ext):
# #
include_dirs = [ include_dirs = [
join(F, fw + '.framework', H) join(F, fw + '.framework', H)
for fw in 'Tcl', 'Tk' for fw in ('Tcl', 'Tk')
for H in 'Headers', 'Versions/Current/PrivateHeaders' for H in ('Headers', 'Versions/Current/PrivateHeaders')
] ]
# For 8.4a2, the X11 headers are not included. Rather than include a # For 8.4a2, the X11 headers are not included. Rather than include a